def setUpClass(cls):
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse3_id = get_rse(cls.rse3).id
cls.rse4_id = get_rse(cls.rse4).id
cls.rse5_id = get_rse(cls.rse5).id
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1, cls.T1, True)
add_rse_attribute(cls.rse3, cls.T1, True)
add_rse_attribute(cls.rse4, cls.T2, True)
add_rse_attribute(cls.rse5, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1, "fakeweight", 10)
add_rse_attribute(cls.rse3, "fakeweight", 0)
add_rse_attribute(cls.rse4, "fakeweight", 0)
add_rse_attribute(cls.rse5, "fakeweight", 0)
def setUp(self):
self.scope = 'mock'
self.rse = 'MOCK4'
self.rse2 = 'MOCK3'
self.account = 'root'
self.rse_id = get_rse(self.rse).id
self.rse_id2 = get_rse(self.rse2).id
self.db_session = session.get_session()
def test_importer_core(self):
""" IMPORTER (CORE): test import. """
import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
# one protocol should be created, one should be updated
check_protocols(self.old_rse_1, self.data1['rses'])
# one protocol should be removed as it is not specified in the import data
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
# RSE 4 should be flagged as deleted as it is missing in the import data
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_data(data=self.data2)
import_data(data=self.data3)
def setUpClass(cls):
# Add test account
cls.account = ''.join(random.choice(string.ascii_uppercase) for x in range(10))
add_account(account=cls.account, type=AccountType.USER)
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse2_id = get_rse(cls.rse2).id
def setUpClass(cls):
# Add test account
cls.account = ''.join(random.choice(string.ascii_uppercase) for x in range(10))
add_account(account=cls.account, type=AccountType.USER, email='*****@*****.**')
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse2 = 'MOCK2'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse2_id = get_rse(cls.rse2).id
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def test_update_and_remove_rse_qos_class(self):
""" QoS (CORE): Update and remove QoS class for RSE """
update_rse(self.tmp_rse, {'qos_class': 'fast_and_expensive'})
rse = get_rse(self.tmp_rse)
assert rse['qos_class'] == 'fast_and_expensive'
update_rse(self.tmp_rse, {'qos_class': 'slow_but_cheap'})
rse = get_rse(self.tmp_rse)
assert rse['qos_class'] == 'slow_but_cheap'
update_rse(self.tmp_rse, {'qos_class': None})
rse = get_rse(self.tmp_rse)
assert rse['qos_class'] is None
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
protocols = get_rse_protocols(self.old_rse_1)
protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']]
assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already existed before
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
with assert_raises(RSENotFound):
get_rse(self.old_rse_3)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3))
assert_equal(r2.status, 201)
def __schedule_requests():
"""
Schedule requests
"""
try:
throttler_mode = config_core.get('throttler',
'mode',
default='DEST_PER_ACT',
use_cache=False)
direction, all_activities = get_parsed_throttler_mode(throttler_mode)
result_dict = __get_request_stats(all_activities, direction)
if direction == 'destination' or direction == 'source':
for rse_id in result_dict:
rse_name = result_dict[rse_id]['rse']
availability = get_rse(rse_id).availability
# dest_rse is not blacklisted for write or src_rse is not blacklisted for read
if (direction == 'destination'
and availability & 2) or (direction == 'source'
and availability & 4):
if all_activities:
__release_all_activities(result_dict[rse_id],
direction, rse_name, rse_id)
else:
__release_per_activity(result_dict[rse_id], direction,
rse_name, rse_id)
except Exception:
logging.critical("Failed to schedule requests, error: %s" %
(traceback.format_exc()))
def run_once(worker_number=0, logger=logging.log, session=None, **kwargs):
"""
Schedule requests
"""
if worker_number != 0:
logger(logging.INFO, 'Throttler thread id is not 0, will sleep. Only thread 0 will work')
return True
logger(logging.INFO, "Throttler - schedule requests")
try:
throttler_mode = config_core.get('throttler', 'mode', default='DEST_PER_ACT', use_cache=False)
direction, all_activities = get_parsed_throttler_mode(throttler_mode)
result_dict = __get_request_stats(all_activities, direction)
if direction == 'destination' or direction == 'source':
for rse_id in result_dict:
rse_name = result_dict[rse_id]['rse']
availability = get_rse(rse_id).availability
# dest_rse is not blocklisted for write or src_rse is not blocklisted for read
if (direction == 'destination' and availability & 2) or (direction == 'source' and availability & 4):
if all_activities:
__release_all_activities(result_dict[rse_id], direction, rse_name, rse_id, logger=logger, session=session)
else:
__release_per_activity(result_dict[rse_id], direction, rse_name, rse_id, logger=logger, session=session)
except Exception:
logger(logging.CRITICAL, "Failed to schedule requests, error: %s" % (traceback.format_exc()))
return True
def test_reaper():
""" REAPER (DAEMON): Test the reaper daemon."""
nb_files = 30
file_size = 2147483648L # 2G
for i in xrange(nb_files):
replica_core.add_replica(rse='MOCK',
scope='data13_hip',
name='lfn' + generate_uuid(),
bytes=file_size,
account='root',
adler32=None,
md5=None)
rse_core.set_rse_usage(rse='MOCK',
source='srm',
used=nb_files * file_size,
free=800L)
rse_core.set_rse_limits(rse='MOCK',
name='MinFreeSpace',
value=10737418240L)
rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10)
rses = [
rse_core.get_rse('MOCK'),
]
reaper(once=True, rses=rses)
reaper(once=True, rses=rses)
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse('MOCK').id
account = 'jdoe'
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in range(10):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(4):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(5):
account_counter.increase(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in range(8):
account_counter.decrease(rse_id=rse_id,
account=account,
files=1,
bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.upload_client = UploadClient()
self.file_sizes = 2
self.rse = 'MOCK4'
self.rse_id = get_rse(self.rse).id
self.session = get_session()
def setUp(self):
self.account = 'root'
self.scope = 'mock'
self.rule_client = RuleClient()
self.did_client = DIDClient()
self.replica_client = ReplicaClient()
self.upload_client = UploadClient()
self.file_sizes = 2
self.dataset = 'dataset_%s' % generate_uuid()
self.rse = 'MOCK5'
self.rse_id = get_rse(self.rse).id
def add_replicas(rse, files, account, rse_id=None, ignore_availability=True, session=None):
"""
Bulk add file replicas.
:param rse: The rse name.
:param files: The list of files.
:param account: The account owner.
:param rse_id: The RSE id. To be used if rse parameter is None.
:param ignore_availability: Ignore the RSE blacklisting.
:param session: The database session in use.
:returns: True is successful.
"""
if rse:
replica_rse = get_rse(rse=rse, session=session)
else:
replica_rse = get_rse(rse=None, rse_id=rse_id, session=session)
if (not (replica_rse.availability & 2)) and not ignore_availability:
raise exception.RessourceTemporaryUnavailable('%s is temporary unavailable for writing' % rse)
replicas = __bulk_add_file_dids(files=files, account=account, session=session)
if not replica_rse.deterministic:
pfns, scheme = list(), None
for file in files:
if 'pfn' not in file:
raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %(rse)s ' % locals())
else:
scheme = file['pfn'].split(':')[0]
pfns.append(file['pfn'])
p = rsemgr.create_protocol(rse_settings=rsemgr.get_rse_info(rse, session=session), operation='write', scheme=scheme)
pfns = p.parse_pfns(pfns=pfns)
for file in files:
tmp = pfns[file['pfn']]
file['path'] = ''.join([tmp['path'], tmp['name']])
nbfiles, bytes = __bulk_add_replicas(rse_id=replica_rse.id, files=files, account=account, session=session)
increase(rse_id=replica_rse.id, files=nbfiles, bytes=bytes, session=session)
return replicas
def check_rse(rse_name, test_data):
rse = get_rse(rse_name)
assert_equal(rse['rse'], rse_name)
assert_equal(rse['rse_type'], test_data[rse_name]['rse_type'])
assert_equal(rse['region_code'], test_data[rse_name]['region_code'])
assert_equal(rse['country_name'], test_data[rse_name]['country_name'])
assert_equal(rse['time_zone'], test_data[rse_name]['time_zone'])
assert_equal(rse['volatile'], test_data[rse_name]['volatile'])
assert_equal(rse['deterministic'], test_data[rse_name]['deterministic'])
assert_equal(rse['city'], test_data[rse_name]['city'])
assert_equal(rse['staging_area'], test_data[rse_name]['staging_area'])
assert_equal(rse['longitude'], test_data[rse_name]['longitude'])
assert_equal(rse['latitude'], test_data[rse_name]['latitude'])
assert_equal(rse['availability'], test_data[rse_name]['availability'])
def test_set_rse_limits(self):
""" RSE (CLIENTS): Test the update of RSE limits."""
nb_files = 30
file_size = 2147483648L # 2G
for file in xrange(nb_files):
replica_core.add_replica(rse='MOCK', scope='data13_hip', name='lfn' + generate_uuid(), bytes=file_size, account='root', adler32=None, md5=None)
rse_core.set_rse_usage(rse='MOCK', source='srm', used=nb_files*file_size, free=800L)
rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240L)
rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10)
rses = [rse_core.get_rse('MOCK'), ]
reaper(once=True, rses=rses)
reaper(once=True, rses=rses)
def setUpClass(cls):
# Add test RSE
cls.rse1 = 'MOCK'
cls.rse3 = 'MOCK3'
cls.rse4 = 'MOCK4'
cls.rse5 = 'MOCK5'
cls.rse1_id = get_rse(cls.rse1).id
cls.rse3_id = get_rse(cls.rse3).id
cls.rse4_id = get_rse(cls.rse4).id
cls.rse5_id = get_rse(cls.rse5).id
# Add Tags
cls.T1 = tag_generator()
cls.T2 = tag_generator()
add_rse_attribute(cls.rse1, cls.T1, True)
add_rse_attribute(cls.rse3, cls.T1, True)
add_rse_attribute(cls.rse4, cls.T2, True)
add_rse_attribute(cls.rse5, cls.T1, True)
# Add fake weights
add_rse_attribute(cls.rse1, "fakeweight", 10)
add_rse_attribute(cls.rse3, "fakeweight", 0)
add_rse_attribute(cls.rse4, "fakeweight", 0)
add_rse_attribute(cls.rse5, "fakeweight", 0)
# Add quota
set_account_limit('jdoe', cls.rse1_id, -1)
set_account_limit('jdoe', cls.rse3_id, -1)
set_account_limit('jdoe', cls.rse4_id, -1)
set_account_limit('jdoe', cls.rse5_id, -1)
set_account_limit('root', cls.rse1_id, -1)
set_account_limit('root', cls.rse3_id, -1)
set_account_limit('root', cls.rse4_id, -1)
set_account_limit('root', cls.rse5_id, -1)
def add_temporary_dids(dids, account, session=None):
"""
Bulk add temporary data identifiers.
:param dids: A list of dids.
:param account: The account owner.
:param session: The database session in use.
"""
temporary_dids, rses = [], {}
for did in dids:
rse = did['rse']
if rse not in rses:
if did.get('rse_id'):
rses[rse] = {'id': did['rse_id']}
else:
replica_rse = get_rse(rse=rse, session=session)
rses[rse] = {'id': replica_rse.id}
if did.get('pfn'):
did['path'] = did['pfn']
# In waiting to properly extract the path
# p = rsemgr.create_protocol(rse_settings=rsemgr.get_rse_info(rse, session=session), operation='write', scheme=scheme)
# if not replica_rse.deterministic:
# pfns = p.parse_pfns(pfns=pfns)
# tmp = pfns[file['pfn']]
# file['path'] = ''.join([tmp['path'], tmp['name']])
temporary_dids.append({
'scope': did['scope'],
'name': did['name'],
'rse_id': rses[rse]['id'],
'path': did.get('path'),
'bytes': did.get('bytes'),
'md5': did.get('md5'),
'adler32': did.get('adler32'),
'guid': did.get('guid'),
'events': did.get('envents'),
'parent_scope': did.get('parent_scope'),
'parent_name': did.get('parent_name'),
'offset': did.get('offset'),
'expired_at': datetime.utcnow()
})
try:
session.bulk_insert_mappings(models.TemporaryDataIdentifier,
temporary_dids)
except:
raise
def test_inc_dec_get_counter(self):
"""ACCOUNT COUNTER (CORE): Increase, decrease and get counter """
account_update(once=True)
rse_id = get_rse('MOCK').id
account = 'jdoe'
account_counter.del_counter(rse_id=rse_id, account=account)
account_counter.add_counter(rse_id=rse_id, account=account)
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': 0, 'bytes': 0})
count, sum = 0, 0
for i in xrange(10):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(4):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(5):
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count += 1
sum += 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
for i in xrange(8):
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=2.147e+9)
account_update(once=True)
count -= 1
sum -= 2.147e+9
cnt = account_counter.get_counter(rse_id=rse_id, account=account)
del cnt['updated_at']
assert_equal(cnt, {'files': count, 'bytes': sum})
def test_reaper():
""" REAPER (DAEMON): Test the reaper daemon."""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo = {}
rse_name = rse_name_generator()
rse_id = rse_core.add_rse(rse_name, **vo)
mock_protocol = {'scheme': 'MOCK',
'hostname': 'localhost',
'port': 123,
'prefix': '/test/reaper',
'impl': 'rucio.rse.protocols.mock.Default',
'domains': {
'lan': {'read': 1,
'write': 1,
'delete': 1},
'wan': {'read': 1,
'write': 1,
'delete': 1}}}
rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
nb_files = 30
file_size = 2147483648 # 2G
file_names = []
for i in range(nb_files):
file_name = 'lfn' + generate_uuid()
file_names.append(file_name)
replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip', **vo),
name=file_name, bytes=file_size,
tombstone=datetime.utcnow() - timedelta(days=1),
account=InternalAccount('root', **vo), adler32=None, md5=None)
rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
rses = [rse_core.get_rse(rse_id), ]
reaper(once=True, rses=rses)
reaper(once=True, rses=rses)
assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip', **vo), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 10)
def handle_one_replica(replica, req_type, rule_id, session=None):
"""
Used by finisher to handle a replica.
:param replica: replica as a dictionary.
:param req_type: Request type: STAGEIN, STAGEOUT, TRANSFER.
:param rule_id: RULE id.
:param session: The database session to use.
:returns commit_or_rollback: Boolean.
"""
try:
replica_core.update_replicas_states([replica], nowait=True, session=session)
if not replica['archived']:
request_core.archive_request(replica['request_id'], session=session)
logging.info("HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state'])))
except (UnsupportedOperation, ReplicaNotFound) as error:
logging.warn("ERROR WHEN HANDLING REQUEST %s DID %s:%s AT RSE %s STATE %s: %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state']), str(error)))
# replica cannot be found. register it and schedule it for deletion
try:
if replica['state'] == ReplicaState.AVAILABLE and replica['request_type'] != RequestType.STAGEIN:
logging.info("Replica cannot be found. Adding a replica %s:%s AT RSE %s with tombstone=utcnow" % (replica['scope'], replica['name'], replica['rse_id']))
rse = rse_core.get_rse(rse=None, rse_id=replica['rse_id'], session=session)
replica_core.add_replica(rse['rse'],
replica['scope'],
replica['name'],
replica['bytes'],
pfn=replica['pfn'] if 'pfn' in replica else None,
account='root', # it will deleted immediately, do we need to get the accurate account from rule?
adler32=replica['adler32'],
tombstone=datetime.datetime.utcnow(),
session=session)
if not replica['archived']:
request_core.archive_request(replica['request_id'], session=session)
logging.info("HANDLED REQUEST %s DID %s:%s AT RSE %s STATE %s" % (replica['request_id'], replica['scope'], replica['name'], replica['rse_id'], str(replica['state'])))
except:
logging.error('Cannot register replica for DID %s:%s at RSE %s - potential dark data' % (replica['scope'],
replica['name'],
replica['rse_id']))
raise
return True
def check_rse(rse_name, test_data):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {'vo': 'tst'}
else:
vo = {}
rse_id = get_rse_id(rse=rse_name, **vo)
rse = get_rse(rse_id=rse_id)
assert_equal(rse['rse'], rse_name)
assert_equal(rse['rse_type'], test_data[rse_name]['rse_type'])
assert_equal(rse['region_code'], test_data[rse_name]['region_code'])
assert_equal(rse['country_name'], test_data[rse_name]['country_name'])
assert_equal(rse['time_zone'], test_data[rse_name]['time_zone'])
assert_equal(rse['volatile'], test_data[rse_name]['volatile'])
assert_equal(rse['deterministic'], test_data[rse_name]['deterministic'])
assert_equal(rse['city'], test_data[rse_name]['city'])
assert_equal(rse['staging_area'], test_data[rse_name]['staging_area'])
assert_equal(rse['longitude'], test_data[rse_name]['longitude'])
assert_equal(rse['latitude'], test_data[rse_name]['latitude'])
assert_equal(rse['availability'], test_data[rse_name]['availability'])
def __schedule_requests():
"""
Schedule requests
"""
try:
logging.info("Throttler retrieve requests statistics")
results = get_stats_by_activity_dest_state(state=[
RequestState.QUEUED, RequestState.SUBMITTING,
RequestState.SUBMITTED, RequestState.WAITING
])
result_dict = {}
for activity, dest_rse_id, account, state, rse, counter in results:
threshold = get_config_limit(activity, dest_rse_id)
if threshold or (counter and (state == RequestState.WAITING)):
if activity not in result_dict:
result_dict[activity] = {}
if dest_rse_id not in result_dict[activity]:
result_dict[activity][dest_rse_id] = {
'waiting': 0,
'transfer': 0,
'threshold': threshold,
'accounts': {},
'rse': rse
}
if account not in result_dict[activity][dest_rse_id][
'accounts']:
result_dict[activity][dest_rse_id]['accounts'][account] = {
'waiting': 0,
'transfer': 0
}
if state == RequestState.WAITING:
result_dict[activity][dest_rse_id]['accounts'][account][
'waiting'] += counter
result_dict[activity][dest_rse_id]['waiting'] += counter
else:
result_dict[activity][dest_rse_id]['accounts'][account][
'transfer'] += counter
result_dict[activity][dest_rse_id]['transfer'] += counter
for activity in result_dict:
for dest_rse_id in result_dict[activity]:
rse_name = result_dict[activity][dest_rse_id]['rse']
availability = get_rse(rse_name).availability
if availability & 2: # dest_rse is not blacklisted for write
threshold = result_dict[activity][dest_rse_id]['threshold']
transfer = result_dict[activity][dest_rse_id]['transfer']
waiting = result_dict[activity][dest_rse_id]['waiting']
if waiting:
logging.debug("Request status for %s at %s: %s" %
(activity, rse_name,
result_dict[activity][dest_rse_id]))
if threshold is None:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse_id %s"
% (threshold, activity, dest_rse_id))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
elif transfer + waiting > threshold:
logging.debug(
"Throttler set limits for activity %s, rse %s" %
(activity, rse_name))
set_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id,
max_transfers=threshold,
transfers=transfer,
waitings=waiting)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.max_transfers'
% (activity, rse_name), threshold)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.transfers'
% (activity, rse_name), transfer)
record_gauge(
'daemons.conveyor.throttler.set_rse_transfer_limits.%s.%s.waitings'
% (activity, rse_name), waiting)
if transfer < 0.8 * threshold:
# release requests on account
nr_accounts = len(
result_dict[activity][dest_rse_id]['accounts'])
if nr_accounts < 1:
nr_accounts = 1
to_release = threshold - transfer
threshold_per_account = math.ceil(threshold /
nr_accounts)
to_release_per_account = math.ceil(to_release /
nr_accounts)
accounts = result_dict[activity][dest_rse_id][
'accounts']
for account in accounts:
if nr_accounts == 1:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release, activity, rse_name,
account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release)
elif accounts[account][
'transfer'] > threshold_per_account:
logging.debug(
"Throttler will not release %s waiting requests for activity %s, rse %s, account %s: It queued more transfers than its share "
% (accounts[account]['waiting'],
activity, rse_name, account))
nr_accounts -= 1
to_release_per_account = math.ceil(
to_release / nr_accounts)
elif accounts[account][
'waiting'] < to_release_per_account:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (accounts[account]['waiting'],
activity, rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=accounts[account]['waiting'])
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
accounts[account]['waiting'])
to_release = to_release - accounts[
account]['waiting']
nr_accounts -= 1
to_release_per_account = math.ceil(
to_release / nr_accounts)
else:
logging.debug(
"Throttler release %s waiting requests for activity %s, rse %s, account %s "
% (to_release_per_account, activity,
rse_name, account))
release_waiting_requests(
rse=None,
activity=activity,
rse_id=dest_rse_id,
account=account,
count=to_release_per_account)
record_gauge(
'daemons.conveyor.throttler.release_waiting_requests.%s.%s.%s'
% (activity, rse_name, account),
to_release_per_account)
to_release = to_release - to_release_per_account
nr_accounts -= 1
else:
logging.debug(
"Throttler has done nothing for activity %s on rse %s (transfer > 0.8 * threshold)"
% (activity, rse_name))
elif waiting > 0:
logging.debug(
"Throttler remove limits(threshold: %s) and release all waiting requests for activity %s, rse %s"
% (threshold, activity, rse_name))
delete_rse_transfer_limits(rse=None,
activity=activity,
rse_id=dest_rse_id)
release_waiting_requests(rse=None,
activity=activity,
rse_id=dest_rse_id)
record_counter(
'daemons.conveyor.throttler.delete_rse_transfer_limits.%s.%s'
% (activity, rse_name))
except Exception:
logging.critical("Failed to schedule requests, error: %s" %
(traceback.format_exc()))
def get_transfer(rse, req, scheme, mock):
src_spacetoken = None
ts = time.time()
sources, metadata = get_sources(rse, scheme, req)
record_timer('daemons.conveyor.submitter.get_sources', (time.time() - ts) * 1000)
logging.debug('Sources for request %s: %s' % (req['request_id'], sources))
if sources is None:
logging.error("Request %s DID %s:%s RSE %s failed to get sources" % (req['request_id'],
req['scope'],
req['name'],
rse['rse']))
return None
filesize = metadata['filesize']
md5 = metadata['md5']
adler32 = metadata['adler32']
ts = time.time()
destinations, dest_spacetoken = get_destinations(rse, scheme, req, sources)
record_timer('daemons.conveyor.submitter.get_destinations', (time.time() - ts) * 1000)
logging.debug('Destinations for request %s: %s' % (req['request_id'], destinations))
if destinations is None:
logging.error("Request %s DID %s:%s RSE %s failed to get destinations" % (req['request_id'],
req['scope'],
req['name'],
rse['rse']))
return None
# Come up with mock sources if necessary
if mock:
tmp_sources = []
for s in sources:
tmp_sources.append((s[0], ':'.join(['mock']+s[1].split(':')[1:])))
sources = tmp_sources
tmp_metadata = {'request_id': req['request_id'],
'scope': req['scope'],
'name': req['name'],
'activity': req['activity'],
'src_rse': sources[0][0],
'dst_rse': rse['rse'],
'dest_rse_id': req['dest_rse_id'],
'filesize': filesize,
'md5': md5,
'adler32': adler32}
if 'previous_attempt_id' in req and req['previous_attempt_id']:
tmp_metadata['previous_attempt_id'] = req['previous_attempt_id']
# Extend the metadata dictionary with request attributes
copy_pin_lifetime, overwrite, bring_online = -1, True, None
if req['request_type'] == RequestType.STAGEIN:
if req['attributes']:
if type(req['attributes']) is dict:
attr = json.loads(json.dumps(req['attributes']))
else:
attr = json.loads(str(req['attributes']))
copy_pin_lifetime = attr.get('lifetime')
overwrite = False
bring_online = 21000
# if the source for transfer is a tape rse, set bring_online
if req['request_type'] == RequestType.TRANSFER\
and rse_core.get_rse(sources[0][0]).rse_type == RSEType.TAPE:
bring_online = 21000
# never overwrite on tape destinations
if req['request_type'] == RequestType.TRANSFER\
and rse_core.get_rse(None, rse_id=req['dest_rse_id']).rse_type == RSEType.TAPE:
overwrite = False
# exclude destination replica from source
source_surls = [s[1] for s in sources]
if req['request_type'] == RequestType.STAGEIN and source_surls.sort() == destinations.sort():
logging.debug('STAGING REQUEST %s - Will not try to ignore equivalent sources' % req['request_id'])
elif req['request_type'] == RequestType.STAGEIN:
logging.debug('STAGING REQUEST %s - Forcing destination to source' % req['request_id'])
destinations = source_surls
else:
new_sources = source_surls
for source_surl in source_surls:
if source_surl in destinations:
logging.info('Excluding source %s for request %s' % (source_surl,
req['request_id']))
new_sources.remove(source_surl)
# make sure we only use one source when bring_online is needed
if bring_online and len(new_sources) > 1:
source_surls = [new_sources[0]]
logging.info('Only using first source %s for bring_online request %s' % (source_surls,
req['request_id']))
if not source_surls:
logging.error('All sources excluded - SKIP REQUEST %s' % req['request_id'])
return
# Sources are properly set, so now we can finally force the source RSE to the destination RSE for STAGEIN
if req['request_type'] == RequestType.STAGEIN:
tmp_metadata['dst_rse'] = sources[0][0]
# get external host
if rse_core.get_rse(rse['rse'])['staging_area'] or rse['rse'].endswith("STAGING"):
rse_attr = rse_core.list_rse_attributes(sources[0][0])
else:
rse_attr = rse_core.list_rse_attributes(rse['rse'], rse['id'])
fts_hosts = rse_attr.get('fts', None)
retry_count = req['retry_count']
if not retry_count:
retry_count = 0
if not fts_hosts:
logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (rse['rse'], req['request_id']))
return
fts_list = fts_hosts.split(",")
external_host = fts_list[retry_count/len(fts_list)]
transfer = {'request_id': req['request_id'],
'src_urls': source_surls,
'dest_urls': destinations,
'filesize': filesize,
'md5': md5,
'adler32': adler32,
'src_spacetoken': src_spacetoken,
'dest_spacetoken': dest_spacetoken,
'activity': req['activity'],
'overwrite': overwrite,
'bring_online': bring_online,
'copy_pin_lifetime': copy_pin_lifetime,
'external_host': external_host,
'file_metadata': tmp_metadata}
return transfer
# return gracefully if there are no replicas for a DID
if not replications:
return None, None
for source in replications:
metadata['filesize'] = long(source['bytes'])
metadata['md5'] = source['md5']
metadata['adler32'] = source['adler32']
# TODO: Source protection
# we need to know upfront if we are mixed DISK/TAPE source
mixed_source = []
for source_rse in source['rses']:
mixed_source.append(rse_core.get_rse(source_rse).rse_type)
mixed_source = True if len(set(mixed_source)) > 1 else False
for source_rse in source['rses']:
if req['request_type'] == RequestType.STAGEIN:
if source_rse in allowed_rses:
for pfn in source['rses'][source_rse]:
# In case of staging request, we only use one source
tmpsrc = [(str(source_rse), str(pfn)), ]
elif req['request_type'] == RequestType.TRANSFER:
if source_rse == dest_rse['rse']:
logging.debug('Skip source %s for request %s because it is the destination' % (source_rse,
req['request_id']))
continue
def submitter(once=False, rses=[], process=0, total_processes=1, thread=0, total_threads=1, mock=False, bulk=100, activities=None):
"""
Main loop to submit a new transfer primitive to a transfertool.
"""
logging.info('submitter starting - process (%i/%i) thread (%i/%i)' % (process,
total_processes,
thread,
total_threads))
try:
scheme = config_get('conveyor', 'scheme')
except NoOptionError:
scheme = 'srm'
logging.info('submitter started - process (%i/%i) thread (%i/%i)' % (process,
total_processes,
thread,
total_threads))
while not graceful_stop.is_set():
try:
if activities is None:
activities = [None]
for activity in activities:
if rses is None:
rses = [None]
for rse in rses:
if rse:
# run in rse list mode
rse_info = rsemgr.get_rse_info(rse['rse'])
logging.info("Working on RSE: %s" % rse['rse'])
ts = time.time()
reqs = get_requests(rse_id=rse['id'],
process=process,
total_processes=total_processes,
thread=thread,
total_threads=total_threads,
mock=mock,
bulk=bulk,
activity=activity)
record_timer('daemons.conveyor.submitter.get_requests', (time.time() - ts) * 1000)
else:
# no rse list, run FIFO mode
rse_info = None
ts = time.time()
reqs = get_requests(process=process,
total_processes=total_processes,
thread=thread,
total_threads=total_threads,
mock=mock,
bulk=bulk,
activity=activity)
record_timer('daemons.conveyor.submitter.get_requests', (time.time() - ts) * 1000)
if reqs:
logging.debug('%i:%i - submitting %i requests' % (process, thread, len(reqs)))
if not reqs or reqs == []:
time.sleep(1)
continue
for req in reqs:
try:
if not rse:
# no rse list, in FIFO mode
dest_rse = rse_core.get_rse(rse=None, rse_id=req['dest_rse_id'])
rse_info = rsemgr.get_rse_info(dest_rse['rse'])
ts = time.time()
transfer = get_transfer(rse_info, req, scheme, mock)
record_timer('daemons.conveyor.submitter.get_transfer', (time.time() - ts) * 1000)
logging.debug('Transfer for request %s: %s' % (req['request_id'], transfer))
if transfer is None:
logging.warn("Request %s DID %s:%s RSE %s failed to get transfer" % (req['request_id'],
req['scope'],
req['name'],
rse_info['rse']))
# TODO: Merge these two calls
request.set_request_state(req['request_id'],
RequestState.LOST) # if the DID does not exist anymore
request.archive_request(req['request_id'])
continue
ts = time.time()
tmp_metadata = transfer['file_metadata']
eids = request.submit_transfers(transfers=[transfer, ],
transfertool='fts3',
job_metadata=tmp_metadata)
record_timer('daemons.conveyor.submitter.submit_transfer', (time.time() - ts) * 1000)
ts = time.time()
if req['previous_attempt_id']:
logging.info('COPYING RETRY %s REQUEST %s PREVIOUS %s DID %s:%s FROM %s TO %s USING %s with eid: %s' % (req['retry_count'],
req['request_id'],
req['previous_attempt_id'],
req['scope'],
req['name'],
transfer['src_urls'],
transfer['dest_urls'],
eids[req['request_id']]['external_host'],
eids[req['request_id']]['external_id']))
else:
logging.info('COPYING REQUEST %s DID %s:%s FROM %s TO %s USING %s with eid: %s' % (req['request_id'],
req['scope'],
req['name'],
transfer['src_urls'],
transfer['dest_urls'],
eids[req['request_id']]['external_host'],
eids[req['request_id']]['external_id']))
record_counter('daemons.conveyor.submitter.submit_request')
except UnsupportedOperation, e:
# The replica doesn't exist, need to cancel the request
logging.warning(e)
logging.info('Cancelling transfer request %s' % req['request_id'])
try:
# TODO: for now, there is only ever one destination
request.cancel_request_did(req['scope'], req['name'], transfer['dest_urls'][0])
except Exception, e:
logging.warning('Cannot cancel request: %s' % str(e))
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=self.data1)
# RSE that had not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already exists
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {
'X-Rucio-Type': 'user',
'X-Rucio-Account': 'root',
'X-Rucio-Auth-Token': str(token)
}
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already existed before
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data3))
assert_equal(r2.status, 201)
def place(self, did):
self.__update_penalties()
self._added_bytes.trim()
self._added_files.trim()
decision = self.check_did(did)
if 'error_reason' in decision:
return decision
meta = get_did(did[0], did[1])
available_reps = {}
reps = list_dataset_replicas(did[0], did[1])
num_reps = 0
space_info = self._fsc.get_rse_space()
max_mbps = 0.0
for rep in reps:
rse_attr = list_rse_attributes(rep['rse'])
src_rse = rep['rse']
if 'site' not in rse_attr:
continue
src_site = rse_attr['site']
src_rse_info = get_rse(src_rse)
if 'type' not in rse_attr:
continue
if rse_attr['type'] != 'DATADISK':
continue
if src_rse_info['availability'] & 4 == 0:
continue
if rep['state'] == ReplicaState.AVAILABLE:
if rep['available_length'] == 0:
continue
net_metrics = {}
net_metrics_type = None
for metric_type in ('fts', 'fax', 'perfsonar', 'dashb'):
net_metrics_type = metric_type
net_metrics = self._nmc.getMbps(src_site, metric_type)
if net_metrics:
break
if len(net_metrics) == 0:
continue
available_reps[src_rse] = {}
for dst_site, mbps in net_metrics.items():
if src_site == dst_site:
continue
if dst_site in self._sites:
if mbps > max_mbps:
max_mbps = mbps
dst_rse = self._sites[dst_site]['rse']
dst_rse_info = get_rse(dst_rse)
if dst_rse_info['availability'] & 2 == 0:
continue
site_added_bytes = sum(self._added_bytes.get_series(dst_rse))
site_added_files = sum(self._added_files.get_series(dst_rse))
if ((site_added_bytes + meta['bytes']) > self._max_bytes_hour_rse):
continue
if ((site_added_files + meta['length']) > self._max_files_hour_rse):
continue
queued = self._nmc.getQueuedFiles(src_site, dst_site)
# logging.debug('queued %s -> %s: %d' % (src_site, dst_site, queued))
if queued > 0:
continue
rse_space = space_info.get(dst_rse, {'free': 0, 'total': 1})
if src_rse not in self._src_penalties:
self._src_penalties[src_rse] = 100.0
src_penalty = self._src_penalties[src_rse]
if dst_rse not in self._dst_penalties:
self._dst_penalties[dst_rse] = 100.0
dst_penalty = self._dst_penalties[dst_rse]
free_space = float(rse_space['free']) / float(rse_space['total']) * 100.0
available_reps[src_rse][dst_rse] = {'free_space': free_space, 'src_penalty': src_penalty, 'dst_penalty': dst_penalty, 'mbps': float(mbps), 'metrics_type': net_metrics_type}
num_reps += 1
# decision['replica_rses'] = available_reps
decision['num_replicas'] = num_reps
if num_reps >= 5:
decision['error_reason'] = 'more than 4 replicas already exist'
return decision
src_dst_ratios = []
if max_mbps == 0.0:
decision['error_reason'] = 'could not find enough network metrics'
return decision
for src, dsts in available_reps.items():
for dst, metrics in dsts.items():
if dst in available_reps:
continue
bdw = (metrics['mbps'] / max_mbps) * 100.0
src_penalty = self._src_penalties[src]
dst_penalty = self._dst_penalties[dst]
ratio = ((metrics['free_space'] / 4.0) + bdw) * src_penalty * dst_penalty
src_dst_ratios.append((src, dst, ratio))
if len(src_dst_ratios) == 0:
decision['error_reason'] = 'found no suitable src/dst for replication'
return decision
sorted_ratios = sorted(src_dst_ratios, key=itemgetter(2), reverse=True)
logging.debug(sorted_ratios)
destination_rse = sorted_ratios[0][1]
source_rse = sorted_ratios[0][0]
decision['destination_rse'] = destination_rse
decision['source_rse'] = source_rse
# decision['rse_ratios'] = src_dst_ratios
self._dst_penalties[destination_rse] = 10.0
self._src_penalties[source_rse] = 10.0
self._added_cache.add_dataset(':'.join(did))
self._added_bytes.add_point(destination_rse, meta['bytes'])
self._added_files.add_point(destination_rse, meta['length'])
self._added_bytes.add_point('total', meta['bytes'])
self._added_files.add_point('total', meta['length'])
return decision
def delete_replicas(rse, files, ignore_availability=True, session=None):
"""
Delete file replicas.
:param rse: the rse name.
:param files: the list of files to delete.
:param ignore_availability: Ignore the RSE blacklisting.
:param session: The database session in use.
"""
replica_rse = get_rse(rse=rse, session=session)
if (not (replica_rse.availability & 1)) and not ignore_availability:
raise exception.RessourceTemporaryUnavailable('%s is temporary unavailable for deleting' % rse)
replica_condition, parent_condition, did_condition = list(), list(), list()
for file in files:
replica_condition.append(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name']))
parent_condition.append(and_(models.DataIdentifierAssociation.child_scope == file['scope'], models.DataIdentifierAssociation.child_name == file['name'],
~exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')).where(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name']))))
did_condition.append(and_(models.DataIdentifier.scope == file['scope'], models.DataIdentifier.name == file['name'],
~exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')).where(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name']))))
delta, bytes, rowcount = 0, 0, 0
for c in chunks(replica_condition, 10):
for (replica_bytes, ) in session.query(models.RSEFileAssociation.bytes).with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_PK)", 'oracle').filter(models.RSEFileAssociation.rse_id == replica_rse.id).filter(or_(*c)):
bytes += replica_bytes
delta += 1
rowcount += session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.rse_id == replica_rse.id).filter(or_(*c)).delete(synchronize_session=False)
if rowcount != len(files):
raise exception.ReplicaNotFound("One or several replicas don't exist.")
# Delete did from the content for the last did
while parent_condition:
child_did_condition = list()
tmp_parent_condition = list()
for c in chunks(parent_condition, 10):
query = session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name,
models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).\
filter(or_(*c))
for parent_scope, parent_name, child_scope, child_name in query:
child_did_condition.append(and_(models.DataIdentifierAssociation.scope == parent_scope, models.DataIdentifierAssociation.name == parent_name,
models.DataIdentifierAssociation.child_scope == child_scope, models.DataIdentifierAssociation.child_name == child_name))
tmp_parent_condition.append(and_(models.DataIdentifierAssociation.child_scope == parent_scope, models.DataIdentifierAssociation.child_name == parent_name,
~exists(select([1]).prefix_with("/*+ INDEX(CONTENTS CONTENTS_PK) */", dialect='oracle')).where(and_(models.DataIdentifierAssociation.scope == parent_scope,
models.DataIdentifierAssociation.name == parent_name))))
did_condition.append(and_(models.DataIdentifier.scope == parent_scope, models.DataIdentifier.name == parent_name, models.DataIdentifier.is_open == False,
~exists([1]).where(and_(models.DataIdentifierAssociation.scope == parent_scope, models.DataIdentifierAssociation.name == parent_name)))) # NOQA
if child_did_condition:
for c in chunks(child_did_condition, 10):
rowcount = session.query(models.DataIdentifierAssociation).filter(or_(*c)).delete(synchronize_session=False)
# update parent counters
parent_condition = tmp_parent_condition
for c in chunks(did_condition, 10):
rowcount = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter(or_(*c)).delete(synchronize_session=False)
# Decrease RSE counter
decrease(rse_id=replica_rse.id, files=delta, bytes=bytes, session=session)
