def setUpClass(cls): # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse(cls.rse1).id cls.rse3_id = get_rse(cls.rse3).id cls.rse4_id = get_rse(cls.rse4).id cls.rse5_id = get_rse(cls.rse5).id # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1, cls.T1, True) add_rse_attribute(cls.rse3, cls.T1, True) add_rse_attribute(cls.rse4, cls.T2, True) add_rse_attribute(cls.rse5, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1, "fakeweight", 10) add_rse_attribute(cls.rse3, "fakeweight", 0) add_rse_attribute(cls.rse4, "fakeweight", 0) add_rse_attribute(cls.rse5, "fakeweight", 0)
def setUp(self): self.scope = 'mock' self.rse = 'MOCK4' self.rse2 = 'MOCK3' self.account = 'root' self.rse_id = get_rse(self.rse).id self.rse_id2 = get_rse(self.rse2).id self.db_session = session.get_session()
def test_importer_core(self): """ IMPORTER (CORE): test import. """ import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) # one protocol should be created, one should be updated check_protocols(self.old_rse_1, self.data1['rses']) # one protocol should be removed as it is not specified in the import data protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) # RSE 4 should be flagged as deleted as it is missing in the import data with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_data(data=self.data2) import_data(data=self.data3)
def setUpClass(cls): # Add test account cls.account = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) add_account(account=cls.account, type=AccountType.USER) # Add test RSE cls.rse1 = 'MOCK' cls.rse2 = 'MOCK2' cls.rse1_id = get_rse(cls.rse1).id cls.rse2_id = get_rse(cls.rse2).id
def setUpClass(cls): # Add test account cls.account = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) add_account(account=cls.account, type=AccountType.USER, email='*****@*****.**') # Add test RSE cls.rse1 = 'MOCK' cls.rse2 = 'MOCK2' cls.rse1_id = get_rse(cls.rse1).id cls.rse2_id = get_rse(cls.rse2).id
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)
def test_update_and_remove_rse_qos_class(self): """ QoS (CORE): Update and remove QoS class for RSE """ update_rse(self.tmp_rse, {'qos_class': 'fast_and_expensive'}) rse = get_rse(self.tmp_rse) assert rse['qos_class'] == 'fast_and_expensive' update_rse(self.tmp_rse, {'qos_class': 'slow_but_cheap'}) rse = get_rse(self.tmp_rse) assert rse['qos_class'] == 'slow_but_cheap' update_rse(self.tmp_rse, {'qos_class': None}) rse = get_rse(self.tmp_rse) assert rse['qos_class'] is None
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)} r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) protocols = get_rse_protocols(self.old_rse_1) protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']] assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already existed before check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) with assert_raises(RSENotFound): get_rse(self.old_rse_3) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)
def __schedule_requests(): """ Schedule requests """ try: throttler_mode = config_core.get('throttler', 'mode', default='DEST_PER_ACT', use_cache=False) direction, all_activities = get_parsed_throttler_mode(throttler_mode) result_dict = __get_request_stats(all_activities, direction) if direction == 'destination' or direction == 'source': for rse_id in result_dict: rse_name = result_dict[rse_id]['rse'] availability = get_rse(rse_id).availability # dest_rse is not blacklisted for write or src_rse is not blacklisted for read if (direction == 'destination' and availability & 2) or (direction == 'source' and availability & 4): if all_activities: __release_all_activities(result_dict[rse_id], direction, rse_name, rse_id) else: __release_per_activity(result_dict[rse_id], direction, rse_name, rse_id) except Exception: logging.critical("Failed to schedule requests, error: %s" % (traceback.format_exc()))
def run_once(worker_number=0, logger=logging.log, session=None, **kwargs): """ Schedule requests """ if worker_number != 0: logger(logging.INFO, 'Throttler thread id is not 0, will sleep. Only thread 0 will work') return True logger(logging.INFO, "Throttler - schedule requests") try: throttler_mode = config_core.get('throttler', 'mode', default='DEST_PER_ACT', use_cache=False) direction, all_activities = get_parsed_throttler_mode(throttler_mode) result_dict = __get_request_stats(all_activities, direction) if direction == 'destination' or direction == 'source': for rse_id in result_dict: rse_name = result_dict[rse_id]['rse'] availability = get_rse(rse_id).availability # dest_rse is not blocklisted for write or src_rse is not blocklisted for read if (direction == 'destination' and availability & 2) or (direction == 'source' and availability & 4): if all_activities: __release_all_activities(result_dict[rse_id], direction, rse_name, rse_id, logger=logger, session=session) else: __release_per_activity(result_dict[rse_id], direction, rse_name, rse_id, logger=logger, session=session) except Exception: logger(logging.CRITICAL, "Failed to schedule requests, error: %s" % (traceback.format_exc())) return True
def test_reaper(): """ REAPER (DAEMON): Test the reaper daemon.""" nb_files = 30 file_size = 2147483648L # 2G for i in xrange(nb_files): replica_core.add_replica(rse='MOCK', scope='data13_hip', name='lfn' + generate_uuid(), bytes=file_size, account='root', adler32=None, md5=None) rse_core.set_rse_usage(rse='MOCK', source='srm', used=nb_files * file_size, free=800L) rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240L) rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10) rses = [ rse_core.get_rse('MOCK'), ] reaper(once=True, rses=rses) reaper(once=True, rses=rses)
def test_inc_dec_get_counter(self): """ACCOUNT COUNTER (CORE): Increase, decrease and get counter """ account_update(once=True) rse_id = get_rse('MOCK').id account = 'jdoe' account_counter.del_counter(rse_id=rse_id, account=account) account_counter.add_counter(rse_id=rse_id, account=account) cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': 0, 'bytes': 0}) count, sum = 0, 0 for i in range(10): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(4): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(5): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in range(8): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum})
def setUp(self): self.account = 'root' self.scope = 'mock' self.upload_client = UploadClient() self.file_sizes = 2 self.rse = 'MOCK4' self.rse_id = get_rse(self.rse).id self.session = get_session()
def setUp(self): self.account = 'root' self.scope = 'mock' self.rule_client = RuleClient() self.did_client = DIDClient() self.replica_client = ReplicaClient() self.upload_client = UploadClient() self.file_sizes = 2 self.dataset = 'dataset_%s' % generate_uuid() self.rse = 'MOCK5' self.rse_id = get_rse(self.rse).id
def add_replicas(rse, files, account, rse_id=None, ignore_availability=True, session=None): """ Bulk add file replicas. :param rse: The rse name. :param files: The list of files. :param account: The account owner. :param rse_id: The RSE id. To be used if rse parameter is None. :param ignore_availability: Ignore the RSE blacklisting. :param session: The database session in use. :returns: True is successful. """ if rse: replica_rse = get_rse(rse=rse, session=session) else: replica_rse = get_rse(rse=None, rse_id=rse_id, session=session) if (not (replica_rse.availability & 2)) and not ignore_availability: raise exception.RessourceTemporaryUnavailable('%s is temporary unavailable for writing' % rse) replicas = __bulk_add_file_dids(files=files, account=account, session=session) if not replica_rse.deterministic: pfns, scheme = list(), None for file in files: if 'pfn' not in file: raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %(rse)s ' % locals()) else: scheme = file['pfn'].split(':')[0] pfns.append(file['pfn']) p = rsemgr.create_protocol(rse_settings=rsemgr.get_rse_info(rse, session=session), operation='write', scheme=scheme) pfns = p.parse_pfns(pfns=pfns) for file in files: tmp = pfns[file['pfn']] file['path'] = ''.join([tmp['path'], tmp['name']]) nbfiles, bytes = __bulk_add_replicas(rse_id=replica_rse.id, files=files, account=account, session=session) increase(rse_id=replica_rse.id, files=nbfiles, bytes=bytes, session=session) return replicas
def check_rse(rse_name, test_data): rse = get_rse(rse_name) assert_equal(rse['rse'], rse_name) assert_equal(rse['rse_type'], test_data[rse_name]['rse_type']) assert_equal(rse['region_code'], test_data[rse_name]['region_code']) assert_equal(rse['country_name'], test_data[rse_name]['country_name']) assert_equal(rse['time_zone'], test_data[rse_name]['time_zone']) assert_equal(rse['volatile'], test_data[rse_name]['volatile']) assert_equal(rse['deterministic'], test_data[rse_name]['deterministic']) assert_equal(rse['city'], test_data[rse_name]['city']) assert_equal(rse['staging_area'], test_data[rse_name]['staging_area']) assert_equal(rse['longitude'], test_data[rse_name]['longitude']) assert_equal(rse['latitude'], test_data[rse_name]['latitude']) assert_equal(rse['availability'], test_data[rse_name]['availability'])
def test_set_rse_limits(self): """ RSE (CLIENTS): Test the update of RSE limits.""" nb_files = 30 file_size = 2147483648L # 2G for file in xrange(nb_files): replica_core.add_replica(rse='MOCK', scope='data13_hip', name='lfn' + generate_uuid(), bytes=file_size, account='root', adler32=None, md5=None) rse_core.set_rse_usage(rse='MOCK', source='srm', used=nb_files*file_size, free=800L) rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240L) rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10) rses = [rse_core.get_rse('MOCK'), ] reaper(once=True, rses=rses) reaper(once=True, rses=rses)
def setUpClass(cls): # Add test RSE cls.rse1 = 'MOCK' cls.rse3 = 'MOCK3' cls.rse4 = 'MOCK4' cls.rse5 = 'MOCK5' cls.rse1_id = get_rse(cls.rse1).id cls.rse3_id = get_rse(cls.rse3).id cls.rse4_id = get_rse(cls.rse4).id cls.rse5_id = get_rse(cls.rse5).id # Add Tags cls.T1 = tag_generator() cls.T2 = tag_generator() add_rse_attribute(cls.rse1, cls.T1, True) add_rse_attribute(cls.rse3, cls.T1, True) add_rse_attribute(cls.rse4, cls.T2, True) add_rse_attribute(cls.rse5, cls.T1, True) # Add fake weights add_rse_attribute(cls.rse1, "fakeweight", 10) add_rse_attribute(cls.rse3, "fakeweight", 0) add_rse_attribute(cls.rse4, "fakeweight", 0) add_rse_attribute(cls.rse5, "fakeweight", 0) # Add quota set_account_limit('jdoe', cls.rse1_id, -1) set_account_limit('jdoe', cls.rse3_id, -1) set_account_limit('jdoe', cls.rse4_id, -1) set_account_limit('jdoe', cls.rse5_id, -1) set_account_limit('root', cls.rse1_id, -1) set_account_limit('root', cls.rse3_id, -1) set_account_limit('root', cls.rse4_id, -1) set_account_limit('root', cls.rse5_id, -1)
def add_temporary_dids(dids, account, session=None): """ Bulk add temporary data identifiers. :param dids: A list of dids. :param account: The account owner. :param session: The database session in use. """ temporary_dids, rses = [], {} for did in dids: rse = did['rse'] if rse not in rses: if did.get('rse_id'): rses[rse] = {'id': did['rse_id']} else: replica_rse = get_rse(rse=rse, session=session) rses[rse] = {'id': replica_rse.id} if did.get('pfn'): did['path'] = did['pfn'] # In waiting to properly extract the path # p = rsemgr.create_protocol(rse_settings=rsemgr.get_rse_info(rse, session=session), operation='write', scheme=scheme) # if not replica_rse.deterministic: # pfns = p.parse_pfns(pfns=pfns) # tmp = pfns[file['pfn']] # file['path'] = ''.join([tmp['path'], tmp['name']]) temporary_dids.append({ 'scope': did['scope'], 'name': did['name'], 'rse_id': rses[rse]['id'], 'path': did.get('path'), 'bytes': did.get('bytes'), 'md5': did.get('md5'), 'adler32': did.get('adler32'), 'guid': did.get('guid'), 'events': did.get('envents'), 'parent_scope': did.get('parent_scope'), 'parent_name': did.get('parent_name'), 'offset': did.get('offset'), 'expired_at': datetime.utcnow() }) try: session.bulk_insert_mappings(models.TemporaryDataIdentifier, temporary_dids) except: raise
def test_inc_dec_get_counter(self): """ACCOUNT COUNTER (CORE): Increase, decrease and get counter """ account_update(once=True) rse_id = get_rse('MOCK').id account = 'jdoe' account_counter.del_counter(rse_id=rse_id, account=account) account_counter.add_counter(rse_id=rse_id, account=account) cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': 0, 'bytes': 0}) count, sum = 0, 0 for i in xrange(10): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(4): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(5): account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count += 1 sum += 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum}) for i in xrange(8): account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9) account_update(once=True) count -= 1 sum -= 2.147e+9 cnt = account_counter.get_counter(rse_id=rse_id, account=account) del cnt['updated_at'] assert_equal(cnt, {'files': count, 'bytes': sum})
def test_reaper(): """ REAPER (DAEMON): Test the reaper daemon.""" if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: vo = {} rse_name = rse_name_generator() rse_id = rse_core.add_rse(rse_name, **vo) mock_protocol = {'scheme': 'MOCK', 'hostname': 'localhost', 'port': 123, 'prefix': '/test/reaper', 'impl': 'rucio.rse.protocols.mock.Default', 'domains': { 'lan': {'read': 1, 'write': 1, 'delete': 1}, 'wan': {'read': 1, 'write': 1, 'delete': 1}}} rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol) nb_files = 30 file_size = 2147483648 # 2G file_names = [] for i in range(nb_files): file_name = 'lfn' + generate_uuid() file_names.append(file_name) replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo), name=file_name, bytes=file_size, tombstone=datetime.utcnow() - timedelta(days=1), account=InternalAccount('root', **vo), adler32=None, md5=None) rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800) rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240) rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10) rses = [rse_core.get_rse(rse_id), ] reaper(once=True, rses=rses) reaper(once=True, rses=rses) assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 10)
def handle_one_replica(replica, req_type, rule_id, session=None): """ Used by finisher to handle a replica. :param replica: replica as a dictionary. :param req_type: Request type: STAGEIN, STAGEOUT, TRANSFER. :param rule_id: RULE id. :param session: The database session to use. :returns commit_or_rollback: Boolean. """ try: replica_core.update_replicas_states([replica], nowait=True, session=session) if not replica['archived']: request_core.archive_request(replica['request_id'], session=session) logging.info("HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']))) except (UnsupportedOperation, ReplicaNotFound) as error: logging.warn("ERROR WHEN HANDLING REQUEST %s DID %s:%s AT RSE %s STATE %s: %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']), str(error))) # replica cannot be found. register it and schedule it for deletion try: if replica['state'] == ReplicaState.AVAILABLE and replica['request_type'] != RequestType.STAGEIN: logging.info("Replica cannot be found. Adding a replica %s:%s AT RSE %s with tombstone=utcnow" % (replica['scope'], replica['name'], replica['rse_id'])) rse = rse_core.get_rse(rse=None, rse_id=replica['rse_id'], session=session) replica_core.add_replica(rse['rse'], replica['scope'], replica['name'], replica['bytes'], pfn=replica['pfn'] if 'pfn' in replica else None, account='root', # it will deleted immediately, do we need to get the accurate account from rule? adler32=replica['adler32'], tombstone=datetime.datetime.utcnow(), session=session) if not replica['archived']: request_core.archive_request(replica['request_id'], session=session) logging.info("HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']))) except: logging.error('Cannot register replica for DID %s:%s at RSE %s - potential dark data' % (replica['scope'], replica['name'], replica['rse_id'])) raise return True
def check_rse(rse_name, test_data): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': 'tst'} else: vo = {} rse_id = get_rse_id(rse=rse_name, **vo) rse = get_rse(rse_id=rse_id) assert_equal(rse['rse'], rse_name) assert_equal(rse['rse_type'], test_data[rse_name]['rse_type']) assert_equal(rse['region_code'], test_data[rse_name]['region_code']) assert_equal(rse['country_name'], test_data[rse_name]['country_name']) assert_equal(rse['time_zone'], test_data[rse_name]['time_zone']) assert_equal(rse['volatile'], test_data[rse_name]['volatile']) assert_equal(rse['deterministic'], test_data[rse_name]['deterministic']) assert_equal(rse['city'], test_data[rse_name]['city']) assert_equal(rse['staging_area'], test_data[rse_name]['staging_area']) assert_equal(rse['longitude'], test_data[rse_name]['longitude']) assert_equal(rse['latitude'], test_data[rse_name]['latitude']) assert_equal(rse['availability'], test_data[rse_name]['availability'])
def __schedule_requests(): """ Schedule requests """ try: logging.info("Throttler retrieve requests statistics") results = get_stats_by_activity_dest_state(state=[ RequestState.QUEUED, RequestState.SUBMITTING, RequestState.SUBMITTED, RequestState.WAITING ]) result_dict = {} for activity, dest_rse_id, account, state, rse, counter in results: threshold = get_config_limit(activity, dest_rse_id) if threshold or (counter and (state == RequestState.WAITING)): if activity not in result_dict: result_dict[activity] = {} if dest_rse_id not in result_dict[activity]: result_dict[activity][dest_rse_id] = { 'waiting': 0, 'transfer': 0, 'threshold': threshold, 'accounts': {}, 'rse': rse } if account not in result_dict[activity][dest_rse_id][ 'accounts']: result_dict[activity][dest_rse_id]['accounts'][account] = { 'waiting': 0, 'transfer': 0 } if state == RequestState.WAITING: result_dict[activity][dest_rse_id]['accounts'][account][ 'waiting'] += counter result_dict[activity][dest_rse_id]['waiting'] += counter else: result_dict[activity][dest_rse_id]['accounts'][account][ 'transfer'] += counter result_dict[activity][dest_rse_id]['transfer'] += counter for activity in result_dict: for dest_rse_id in result_dict[activity]: rse_name = result_dict[activity][dest_rse_id]['rse'] availability = get_rse(rse_name).availability if availability & 2: # dest_rse is not blacklisted for write threshold = result_dict[activity][dest_rse_id]['threshold'] transfer = result_dict[activity][dest_rse_id]['transfer'] waiting = result_dict[activity][dest_rse_id]['waiting'] if waiting: logging.debug("Request status for %s at %s: %s" % (activity, rse_name, result_dict[activity][dest_rse_id])) if threshold is None: logging.debug( "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s" % (threshold, activity, dest_rse_id)) delete_rse_transfer_limits(rse=None, activity=activity, rse_id=dest_rse_id) release_waiting_requests(rse=None, activity=activity, rse_id=dest_rse_id) record_counter( 'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s' % (activity, rse_name)) elif transfer + waiting > threshold: logging.debug( "Throttler set limits for activity %s, rse %s" % (activity, rse_name)) set_rse_transfer_limits(rse=None, activity=activity, rse_id=dest_rse_id, max_transfers=threshold, transfers=transfer, waitings=waiting) record_gauge( 'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.max_transfers' % (activity, rse_name), threshold) record_gauge( 'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.transfers' % (activity, rse_name), transfer) record_gauge( 'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.waitings' % (activity, rse_name), waiting) if transfer < 0.8 * threshold: # release requests on account nr_accounts = len( result_dict[activity][dest_rse_id]['accounts']) if nr_accounts < 1: nr_accounts = 1 to_release = threshold - transfer threshold_per_account = math.ceil(threshold / nr_accounts) to_release_per_account = math.ceil(to_release / nr_accounts) accounts = result_dict[activity][dest_rse_id][ 'accounts'] for account in accounts: if nr_accounts == 1: logging.debug( "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release, activity, rse_name, account)) release_waiting_requests( rse=None, activity=activity, rse_id=dest_rse_id, account=account, count=to_release) record_gauge( 'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s' % (activity, rse_name, account), to_release) elif accounts[account][ 'transfer'] > threshold_per_account: logging.debug( "Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share " % (accounts[account]['waiting'], activity, rse_name, account)) nr_accounts -= 1 to_release_per_account = math.ceil( to_release / nr_accounts) elif accounts[account][ 'waiting'] < to_release_per_account: logging.debug( "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (accounts[account]['waiting'], activity, rse_name, account)) release_waiting_requests( rse=None, activity=activity, rse_id=dest_rse_id, account=account, count=accounts[account]['waiting']) record_gauge( 'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s' % (activity, rse_name, account), accounts[account]['waiting']) to_release = to_release - accounts[ account]['waiting'] nr_accounts -= 1 to_release_per_account = math.ceil( to_release / nr_accounts) else: logging.debug( "Throttler release %s waiting requests for activity %s, rse %s, account %s " % (to_release_per_account, activity, rse_name, account)) release_waiting_requests( rse=None, activity=activity, rse_id=dest_rse_id, account=account, count=to_release_per_account) record_gauge( 'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s' % (activity, rse_name, account), to_release_per_account) to_release = to_release - to_release_per_account nr_accounts -= 1 else: logging.debug( "Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)" % (activity, rse_name)) elif waiting > 0: logging.debug( "Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s" % (threshold, activity, rse_name)) delete_rse_transfer_limits(rse=None, activity=activity, rse_id=dest_rse_id) release_waiting_requests(rse=None, activity=activity, rse_id=dest_rse_id) record_counter( 'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s' % (activity, rse_name)) except Exception: logging.critical("Failed to schedule requests, error: %s" % (traceback.format_exc()))
def get_transfer(rse, req, scheme, mock): src_spacetoken = None ts = time.time() sources, metadata = get_sources(rse, scheme, req) record_timer('daemons.conveyor.submitter.get_sources', (time.time() - ts) * 1000) logging.debug('Sources for request %s: %s' % (req['request_id'], sources)) if sources is None: logging.error("Request %s DID %s:%s RSE %s failed to get sources" % (req['request_id'], req['scope'], req['name'], rse['rse'])) return None filesize = metadata['filesize'] md5 = metadata['md5'] adler32 = metadata['adler32'] ts = time.time() destinations, dest_spacetoken = get_destinations(rse, scheme, req, sources) record_timer('daemons.conveyor.submitter.get_destinations', (time.time() - ts) * 1000) logging.debug('Destinations for request %s: %s' % (req['request_id'], destinations)) if destinations is None: logging.error("Request %s DID %s:%s RSE %s failed to get destinations" % (req['request_id'], req['scope'], req['name'], rse['rse'])) return None # Come up with mock sources if necessary if mock: tmp_sources = [] for s in sources: tmp_sources.append((s[0], ':'.join(['mock']+s[1].split(':')[1:]))) sources = tmp_sources tmp_metadata = {'request_id': req['request_id'], 'scope': req['scope'], 'name': req['name'], 'activity': req['activity'], 'src_rse': sources[0][0], 'dst_rse': rse['rse'], 'dest_rse_id': req['dest_rse_id'], 'filesize': filesize, 'md5': md5, 'adler32': adler32} if 'previous_attempt_id' in req and req['previous_attempt_id']: tmp_metadata['previous_attempt_id'] = req['previous_attempt_id'] # Extend the metadata dictionary with request attributes copy_pin_lifetime, overwrite, bring_online = -1, True, None if req['request_type'] == RequestType.STAGEIN: if req['attributes']: if type(req['attributes']) is dict: attr = json.loads(json.dumps(req['attributes'])) else: attr = json.loads(str(req['attributes'])) copy_pin_lifetime = attr.get('lifetime') overwrite = False bring_online = 21000 # if the source for transfer is a tape rse, set bring_online if req['request_type'] == RequestType.TRANSFER\ and rse_core.get_rse(sources[0][0]).rse_type == RSEType.TAPE: bring_online = 21000 # never overwrite on tape destinations if req['request_type'] == RequestType.TRANSFER\ and rse_core.get_rse(None, rse_id=req['dest_rse_id']).rse_type == RSEType.TAPE: overwrite = False # exclude destination replica from source source_surls = [s[1] for s in sources] if req['request_type'] == RequestType.STAGEIN and source_surls.sort() == destinations.sort(): logging.debug('STAGING REQUEST %s - Will not try to ignore equivalent sources' % req['request_id']) elif req['request_type'] == RequestType.STAGEIN: logging.debug('STAGING REQUEST %s - Forcing destination to source' % req['request_id']) destinations = source_surls else: new_sources = source_surls for source_surl in source_surls: if source_surl in destinations: logging.info('Excluding source %s for request %s' % (source_surl, req['request_id'])) new_sources.remove(source_surl) # make sure we only use one source when bring_online is needed if bring_online and len(new_sources) > 1: source_surls = [new_sources[0]] logging.info('Only using first source %s for bring_online request %s' % (source_surls, req['request_id'])) if not source_surls: logging.error('All sources excluded - SKIP REQUEST %s' % req['request_id']) return # Sources are properly set, so now we can finally force the source RSE to the destination RSE for STAGEIN if req['request_type'] == RequestType.STAGEIN: tmp_metadata['dst_rse'] = sources[0][0] # get external host if rse_core.get_rse(rse['rse'])['staging_area'] or rse['rse'].endswith("STAGING"): rse_attr = rse_core.list_rse_attributes(sources[0][0]) else: rse_attr = rse_core.list_rse_attributes(rse['rse'], rse['id']) fts_hosts = rse_attr.get('fts', None) retry_count = req['retry_count'] if not retry_count: retry_count = 0 if not fts_hosts: logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (rse['rse'], req['request_id'])) return fts_list = fts_hosts.split(",") external_host = fts_list[retry_count/len(fts_list)] transfer = {'request_id': req['request_id'], 'src_urls': source_surls, 'dest_urls': destinations, 'filesize': filesize, 'md5': md5, 'adler32': adler32, 'src_spacetoken': src_spacetoken, 'dest_spacetoken': dest_spacetoken, 'activity': req['activity'], 'overwrite': overwrite, 'bring_online': bring_online, 'copy_pin_lifetime': copy_pin_lifetime, 'external_host': external_host, 'file_metadata': tmp_metadata} return transfer
# return gracefully if there are no replicas for a DID if not replications: return None, None for source in replications: metadata['filesize'] = long(source['bytes']) metadata['md5'] = source['md5'] metadata['adler32'] = source['adler32'] # TODO: Source protection # we need to know upfront if we are mixed DISK/TAPE source mixed_source = [] for source_rse in source['rses']: mixed_source.append(rse_core.get_rse(source_rse).rse_type) mixed_source = True if len(set(mixed_source)) > 1 else False for source_rse in source['rses']: if req['request_type'] == RequestType.STAGEIN: if source_rse in allowed_rses: for pfn in source['rses'][source_rse]: # In case of staging request, we only use one source tmpsrc = [(str(source_rse), str(pfn)), ] elif req['request_type'] == RequestType.TRANSFER: if source_rse == dest_rse['rse']: logging.debug('Skip source %s for request %s because it is the destination' % (source_rse, req['request_id'])) continue
def submitter(once=False, rses=[], process=0, total_processes=1, thread=0, total_threads=1, mock=False, bulk=100, activities=None): """ Main loop to submit a new transfer primitive to a transfertool. """ logging.info('submitter starting - process (%i/%i) thread (%i/%i)' % (process, total_processes, thread, total_threads)) try: scheme = config_get('conveyor', 'scheme') except NoOptionError: scheme = 'srm' logging.info('submitter started - process (%i/%i) thread (%i/%i)' % (process, total_processes, thread, total_threads)) while not graceful_stop.is_set(): try: if activities is None: activities = [None] for activity in activities: if rses is None: rses = [None] for rse in rses: if rse: # run in rse list mode rse_info = rsemgr.get_rse_info(rse['rse']) logging.info("Working on RSE: %s" % rse['rse']) ts = time.time() reqs = get_requests(rse_id=rse['id'], process=process, total_processes=total_processes, thread=thread, total_threads=total_threads, mock=mock, bulk=bulk, activity=activity) record_timer('daemons.conveyor.submitter.get_requests', (time.time() - ts) * 1000) else: # no rse list, run FIFO mode rse_info = None ts = time.time() reqs = get_requests(process=process, total_processes=total_processes, thread=thread, total_threads=total_threads, mock=mock, bulk=bulk, activity=activity) record_timer('daemons.conveyor.submitter.get_requests', (time.time() - ts) * 1000) if reqs: logging.debug('%i:%i - submitting %i requests' % (process, thread, len(reqs))) if not reqs or reqs == []: time.sleep(1) continue for req in reqs: try: if not rse: # no rse list, in FIFO mode dest_rse = rse_core.get_rse(rse=None, rse_id=req['dest_rse_id']) rse_info = rsemgr.get_rse_info(dest_rse['rse']) ts = time.time() transfer = get_transfer(rse_info, req, scheme, mock) record_timer('daemons.conveyor.submitter.get_transfer', (time.time() - ts) * 1000) logging.debug('Transfer for request %s: %s' % (req['request_id'], transfer)) if transfer is None: logging.warn("Request %s DID %s:%s RSE %s failed to get transfer" % (req['request_id'], req['scope'], req['name'], rse_info['rse'])) # TODO: Merge these two calls request.set_request_state(req['request_id'], RequestState.LOST) # if the DID does not exist anymore request.archive_request(req['request_id']) continue ts = time.time() tmp_metadata = transfer['file_metadata'] eids = request.submit_transfers(transfers=[transfer, ], transfertool='fts3', job_metadata=tmp_metadata) record_timer('daemons.conveyor.submitter.submit_transfer', (time.time() - ts) * 1000) ts = time.time() if req['previous_attempt_id']: logging.info('COPYING RETRY %s REQUEST %s PREVIOUS %s DID %s:%s FROM %s TO %s USING %s with eid: %s' % (req['retry_count'], req['request_id'], req['previous_attempt_id'], req['scope'], req['name'], transfer['src_urls'], transfer['dest_urls'], eids[req['request_id']]['external_host'], eids[req['request_id']]['external_id'])) else: logging.info('COPYING REQUEST %s DID %s:%s FROM %s TO %s USING %s with eid: %s' % (req['request_id'], req['scope'], req['name'], transfer['src_urls'], transfer['dest_urls'], eids[req['request_id']]['external_host'], eids[req['request_id']]['external_id'])) record_counter('daemons.conveyor.submitter.submit_request') except UnsupportedOperation, e: # The replica doesn't exist, need to cancel the request logging.warning(e) logging.info('Cancelling transfer request %s' % req['request_id']) try: # TODO: for now, there is only ever one destination request.cancel_request_did(req['scope'], req['name'], transfer['dest_urls'][0]) except Exception, e: logging.warning('Cannot cancel request: %s' % str(e))
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=self.data1) # RSE that had not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already exists rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = { 'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******' } r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = { 'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token) } r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already existed before rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)
def place(self, did): self.__update_penalties() self._added_bytes.trim() self._added_files.trim() decision = self.check_did(did) if 'error_reason' in decision: return decision meta = get_did(did[0], did[1]) available_reps = {} reps = list_dataset_replicas(did[0], did[1]) num_reps = 0 space_info = self._fsc.get_rse_space() max_mbps = 0.0 for rep in reps: rse_attr = list_rse_attributes(rep['rse']) src_rse = rep['rse'] if 'site' not in rse_attr: continue src_site = rse_attr['site'] src_rse_info = get_rse(src_rse) if 'type' not in rse_attr: continue if rse_attr['type'] != 'DATADISK': continue if src_rse_info['availability'] & 4 == 0: continue if rep['state'] == ReplicaState.AVAILABLE: if rep['available_length'] == 0: continue net_metrics = {} net_metrics_type = None for metric_type in ('fts', 'fax', 'perfsonar', 'dashb'): net_metrics_type = metric_type net_metrics = self._nmc.getMbps(src_site, metric_type) if net_metrics: break if len(net_metrics) == 0: continue available_reps[src_rse] = {} for dst_site, mbps in net_metrics.items(): if src_site == dst_site: continue if dst_site in self._sites: if mbps > max_mbps: max_mbps = mbps dst_rse = self._sites[dst_site]['rse'] dst_rse_info = get_rse(dst_rse) if dst_rse_info['availability'] & 2 == 0: continue site_added_bytes = sum(self._added_bytes.get_series(dst_rse)) site_added_files = sum(self._added_files.get_series(dst_rse)) if ((site_added_bytes + meta['bytes']) > self._max_bytes_hour_rse): continue if ((site_added_files + meta['length']) > self._max_files_hour_rse): continue queued = self._nmc.getQueuedFiles(src_site, dst_site) # logging.debug('queued %s -> %s: %d' % (src_site, dst_site, queued)) if queued > 0: continue rse_space = space_info.get(dst_rse, {'free': 0, 'total': 1}) if src_rse not in self._src_penalties: self._src_penalties[src_rse] = 100.0 src_penalty = self._src_penalties[src_rse] if dst_rse not in self._dst_penalties: self._dst_penalties[dst_rse] = 100.0 dst_penalty = self._dst_penalties[dst_rse] free_space = float(rse_space['free']) / float(rse_space['total']) * 100.0 available_reps[src_rse][dst_rse] = {'free_space': free_space, 'src_penalty': src_penalty, 'dst_penalty': dst_penalty, 'mbps': float(mbps), 'metrics_type': net_metrics_type} num_reps += 1 # decision['replica_rses'] = available_reps decision['num_replicas'] = num_reps if num_reps >= 5: decision['error_reason'] = 'more than 4 replicas already exist' return decision src_dst_ratios = [] if max_mbps == 0.0: decision['error_reason'] = 'could not find enough network metrics' return decision for src, dsts in available_reps.items(): for dst, metrics in dsts.items(): if dst in available_reps: continue bdw = (metrics['mbps'] / max_mbps) * 100.0 src_penalty = self._src_penalties[src] dst_penalty = self._dst_penalties[dst] ratio = ((metrics['free_space'] / 4.0) + bdw) * src_penalty * dst_penalty src_dst_ratios.append((src, dst, ratio)) if len(src_dst_ratios) == 0: decision['error_reason'] = 'found no suitable src/dst for replication' return decision sorted_ratios = sorted(src_dst_ratios, key=itemgetter(2), reverse=True) logging.debug(sorted_ratios) destination_rse = sorted_ratios[0][1] source_rse = sorted_ratios[0][0] decision['destination_rse'] = destination_rse decision['source_rse'] = source_rse # decision['rse_ratios'] = src_dst_ratios self._dst_penalties[destination_rse] = 10.0 self._src_penalties[source_rse] = 10.0 self._added_cache.add_dataset(':'.join(did)) self._added_bytes.add_point(destination_rse, meta['bytes']) self._added_files.add_point(destination_rse, meta['length']) self._added_bytes.add_point('total', meta['bytes']) self._added_files.add_point('total', meta['length']) return decision
def delete_replicas(rse, files, ignore_availability=True, session=None): """ Delete file replicas. :param rse: the rse name. :param files: the list of files to delete. :param ignore_availability: Ignore the RSE blacklisting. :param session: The database session in use. """ replica_rse = get_rse(rse=rse, session=session) if (not (replica_rse.availability & 1)) and not ignore_availability: raise exception.RessourceTemporaryUnavailable('%s is temporary unavailable for deleting' % rse) replica_condition, parent_condition, did_condition = list(), list(), list() for file in files: replica_condition.append(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name'])) parent_condition.append(and_(models.DataIdentifierAssociation.child_scope == file['scope'], models.DataIdentifierAssociation.child_name == file['name'], ~exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')).where(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name'])))) did_condition.append(and_(models.DataIdentifier.scope == file['scope'], models.DataIdentifier.name == file['name'], ~exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')).where(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name'])))) delta, bytes, rowcount = 0, 0, 0 for c in chunks(replica_condition, 10): for (replica_bytes, ) in session.query(models.RSEFileAssociation.bytes).with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_PK)", 'oracle').filter(models.RSEFileAssociation.rse_id == replica_rse.id).filter(or_(*c)): bytes += replica_bytes delta += 1 rowcount += session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.rse_id == replica_rse.id).filter(or_(*c)).delete(synchronize_session=False) if rowcount != len(files): raise exception.ReplicaNotFound("One or several replicas don't exist.") # Delete did from the content for the last did while parent_condition: child_did_condition = list() tmp_parent_condition = list() for c in chunks(parent_condition, 10): query = session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name, models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).\ filter(or_(*c)) for parent_scope, parent_name, child_scope, child_name in query: child_did_condition.append(and_(models.DataIdentifierAssociation.scope == parent_scope, models.DataIdentifierAssociation.name == parent_name, models.DataIdentifierAssociation.child_scope == child_scope, models.DataIdentifierAssociation.child_name == child_name)) tmp_parent_condition.append(and_(models.DataIdentifierAssociation.child_scope == parent_scope, models.DataIdentifierAssociation.child_name == parent_name, ~exists(select([1]).prefix_with("/*+ INDEX(CONTENTS CONTENTS_PK) */", dialect='oracle')).where(and_(models.DataIdentifierAssociation.scope == parent_scope, models.DataIdentifierAssociation.name == parent_name)))) did_condition.append(and_(models.DataIdentifier.scope == parent_scope, models.DataIdentifier.name == parent_name, models.DataIdentifier.is_open == False, ~exists([1]).where(and_(models.DataIdentifierAssociation.scope == parent_scope, models.DataIdentifierAssociation.name == parent_name)))) # NOQA if child_did_condition: for c in chunks(child_did_condition, 10): rowcount = session.query(models.DataIdentifierAssociation).filter(or_(*c)).delete(synchronize_session=False) # update parent counters parent_condition = tmp_parent_condition for c in chunks(did_condition, 10): rowcount = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter(or_(*c)).delete(synchronize_session=False) # Decrease RSE counter decrease(rse_id=replica_rse.id, files=delta, bytes=bytes, session=session)