def test_importer_core(self):
""" IMPORTER (CORE): test import. """
import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
# one protocol should be created, one should be updated
check_protocols(self.old_rse_1, self.data1['rses'])
# one protocol should be removed as it is not specified in the import data
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
# RSE 4 should be flagged as deleted as it is missing in the import data
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_data(data=self.data2)
import_data(data=self.data3)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
protocols = get_rse_protocols(self.old_rse_1)
protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']]
assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already existed before
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
with assert_raises(RSENotFound):
get_rse(self.old_rse_3)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3))
assert_equal(r2.status, 201)
def __check_rse_usage(rse, rse_id):
"""
Internal method to check RSE usage and limits.
:param rse_id: the rse name.
:param rse_id: the rse id.
:returns : max_being_deleted_files, needed_free_space, used, free.
"""
max_being_deleted_files, needed_free_space, used, free = None, None, None, None
# Get RSE limits
limits = rse_core.get_rse_limits(rse=rse, rse_id=rse_id)
if not limits and 'MinFreeSpace' not in limits and 'MaxBeingDeletedFiles' not in limits:
return max_being_deleted_files, needed_free_space, used, free
min_free_space = limits.get('MinFreeSpace')
max_being_deleted_files = limits.get('MaxBeingDeletedFiles')
# Check from which sources to get used and total spaces
# Default is storage
source_for_total_space, source_for_used_space = 'storage', 'storage'
values = get_rse_attribute(rse_id=rse_id, key='sourceForTotalSpace')
if values:
source_for_total_space = values[0]
values = get_rse_attribute(rse_id=rse_id, key='sourceForUsedSpace')
if values:
source_for_used_space = values[0]
logging.debug(
'RSE: %(rse)s, sourceForTotalSpace: %(source_for_total_space)s, '
'sourceForUsedSpace: %(source_for_used_space)s' % locals())
# Get total and used space
usage = rse_core.get_rse_usage(rse=rse,
rse_id=rse_id,
source=source_for_total_space)
if not usage:
return max_being_deleted_files, needed_free_space, used, free
for var in usage:
total, used = var['total'], var['used']
break
if source_for_total_space != source_for_used_space:
usage = rse_core.get_rse_usage(rse=rse,
rse_id=rse_id,
source=source_for_used_space)
if not usage:
return max_being_deleted_files, needed_free_space, None, free
for var in usage:
used = var['used']
break
free = total - used
if min_free_space:
needed_free_space = min_free_space - free
return max_being_deleted_files, needed_free_space, used, free
def get_rse_limits(rse, issuer):
"""
Get RSE limits.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: True if successful, otherwise false.
"""
return rse_module.get_rse_limits(rse=rse)
def get_rse_limits(rse, issuer):
"""
Get RSE limits.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: True if successful, otherwise false.
"""
return rse_module.get_rse_limits(rse=rse)
def get_rse_limits(rse, issuer, vo='def'):
"""
Get RSE limits.
:param rse: The RSE name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:returns: True if successful, otherwise false.
"""
rse_id = rse_module.get_rse_id(rse=rse, vo=vo)
return rse_module.get_rse_limits(rse_id=rse_id)
def get_rse_limits(rse, issuer, vo='def', session=None):
"""
Get RSE limits.
:param rse: The RSE name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: True if successful, otherwise false.
"""
rse_id = rse_module.get_rse_id(rse=rse, vo=vo, session=session)
return rse_module.get_rse_limits(rse_id=rse_id, session=session)
def __check_rse_usage(rse, rse_id):
"""
Internal method to check RSE usage and limits.
:param rse_id: the rse name.
:param rse_id: the rse id.
:returns : max_being_deleted_files, needed_free_space, used, free.
"""
max_being_deleted_files, needed_free_space, used, free = None, None, None, None
# Get RSE limits
limits = rse_core.get_rse_limits(rse=rse, rse_id=rse_id)
if not limits and 'MinFreeSpace' not in limits and 'MaxBeingDeletedFiles' not in limits:
return max_being_deleted_files, needed_free_space, used, free
min_free_space = limits.get('MinFreeSpace')
max_being_deleted_files = limits.get('MaxBeingDeletedFiles')
# Get total space available
usage = rse_core.get_rse_usage(rse=rse, rse_id=rse_id, source='srm')
if not usage:
return max_being_deleted_files, needed_free_space, used, free
for u in usage:
total = u['total']
break
# Get current used space
cnt = get_counter(rse_id=rse_id)
if not cnt:
return max_being_deleted_files, needed_free_space, used, free
used = cnt['bytes']
# Get current amount of bytes and files waiting for deletion
# being_deleted = rse_core.get_sum_count_being_deleted(rse_id=rse_id)
free = total - used
needed_free_space = min_free_space - free
return max_being_deleted_files, needed_free_space, used, free
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def __check_rse_usage(rse, rse_id, prepend_str):
"""
Internal method to check RSE usage and limits.
:param rse_id: the rse name.
:param rse_id: the rse id.
:returns : max_being_deleted_files, needed_free_space, used, free.
"""
result = REGION.get('rse_usage_%s' % rse_id)
if result is NO_VALUE:
max_being_deleted_files, needed_free_space, used, free, obsolete = None, None, None, None, None
# Get RSE limits
limits = get_rse_limits(rse_id=rse_id)
if not limits and 'MinFreeSpace' not in limits and 'MaxBeingDeletedFiles' not in limits:
result = (max_being_deleted_files, needed_free_space, used, free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
min_free_space = limits.get('MinFreeSpace')
max_being_deleted_files = limits.get('MaxBeingDeletedFiles')
# Check from which sources to get used and total spaces
# Default is storage
attributes = list_rse_attributes(rse_id=rse_id)
source_for_total_space = attributes.get('sourceForTotalSpace',
'storage')
source_for_used_space = attributes.get('sourceForUsedSpace', 'storage')
greedy = attributes.get('greedyDeletion', False)
logging.debug(
'%s RSE: %s, source_for_total_space: %s, source_for_used_space: %s',
prepend_str, rse, source_for_total_space, source_for_used_space)
# First of all check if greedy mode is enabled for this RSE
if greedy:
result = (max_being_deleted_files, 1000000000000, used, free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
# Get total, used and obsolete space
rse_usage = get_rse_usage(rse_id=rse_id)
usage = [entry for entry in rse_usage if entry['source'] == 'obsolete']
for var in usage:
obsolete = var['used']
break
usage = [
entry for entry in rse_usage
if entry['source'] == source_for_total_space
]
# If no information is available about disk space, do nothing except if there are replicas with Epoch tombstone
if not usage:
if not obsolete:
result = (max_being_deleted_files, needed_free_space, used,
free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
result = (max_being_deleted_files, obsolete, used, free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
# Extract the total and used space
for var in usage:
total, used = var['total'], var['used']
break
if source_for_total_space != source_for_used_space:
usage = [
entry for entry in rse_usage
if entry['source'] == source_for_used_space
]
if not usage:
result = (max_being_deleted_files, needed_free_space, None,
free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
for var in usage:
used = var['used']
break
free = total - used
if min_free_space:
needed_free_space = min_free_space - free
# If needed_free_space negative, nothing to delete except if some Epoch tombstoned replicas
if needed_free_space <= 0:
needed_free_space = 0 or obsolete
result = (max_being_deleted_files, needed_free_space, used, free)
REGION.set('rse_usage_%s' % rse_id, result)
return result
logging.debug('%s Using cached value for RSE usage on RSE %s', prepend_str,
rse)
return result
def __init__(self,
account,
rses,
weight,
copies,
ignore_account_limit=False,
session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param ignore_account_limit: Flag if the quota should be ignored.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse_id=rse['id'],
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({
'rse_id': rse['id'],
'weight': float(attributes[weight]),
'mock_rse': attributes.get('mock', False),
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
except ValueError:
raise InvalidRuleWeight(
'The RSE \'%s\' has a non-number specified for the weight \'%s\''
% (rse['rse'], weight))
else:
for rse in rses:
mock_rse = has_rse_attribute(rse['id'],
'mock',
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
self.rses.append({
'rse_id': rse['id'],
'weight': 1,
'mock_rse': mock_rse,
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs(
'Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)'
% (self.copies, len(self.rses)))
rses_with_enough_quota = []
if has_account_attribute(account=account, key='admin',
session=session) or ignore_account_limit:
for rse in self.rses:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
rses_with_enough_quota.append(rse)
else:
global_quota_limit = get_global_account_limits(account=account,
session=session)
all_rse_usages = {
usage['rse_id']: usage['bytes']
for usage in get_all_rse_usages_per_account(account=account,
session=session)
}
for rse in self.rses:
if rse['mock_rse']:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
rses_with_enough_quota.append(rse)
else:
# check local quota
local_quota_left = None
quota_limit = get_local_account_limit(account=account,
rse_id=rse['rse_id'],
session=session)
if quota_limit is None:
local_quota_left = 0
else:
local_quota_left = quota_limit - get_usage(
rse_id=rse['rse_id'],
account=account,
session=session)['bytes']
# check global quota
rse['global_quota_left'] = {}
all_global_quota_enough = True
for rse_expression, limit in global_quota_limit.items():
if rse['rse_id'] in limit['resolved_rse_ids']:
quota_limit = limit['limit']
global_quota_left = None
if quota_limit is None:
global_quota_left = 0
else:
rse_expression_usage = 0
for rse_id in limit['resolved_rse_ids']:
rse_expression_usage += all_rse_usages.get(
rse_id, 0)
global_quota_left = quota_limit - rse_expression_usage
if global_quota_left <= 0:
all_global_quota_enough = False
break
else:
rse['global_quota_left'][
rse_expression] = global_quota_left
if local_quota_left > 0 and all_global_quota_enough:
rse['quota_left'] = local_quota_left
space_limit = get_rse_limits(
name='MaxSpaceAvailable',
rse_id=rse['rse_id'],
session=session).get('MaxSpaceAvailable')
if space_limit is None or space_limit < 0:
rse['space_left'] = float('inf')
else:
rse['space_left'] = space_limit - get_rse_counter(
rse_id=rse['rse_id'], session=session)['bytes']
rses_with_enough_quota.append(rse)
self.rses = rses_with_enough_quota
if len(self.rses) < self.copies:
raise InsufficientAccountLimit(
'There is insufficient quota on any of the target RSE\'s to fullfill the operation.'
)
def import_rses(rses, vo='def', session=None):
new_rses = []
for rse_name in rses:
rse = rses[rse_name]
if isinstance(rse.get('rse_type'), string_types):
rse['rse_type'] = RSEType.from_string(str(rse['rse_type']))
try:
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
session=session)
except RSENotFound:
rse_id = rse_module.add_rse(rse=rse_name,
vo=vo,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_id=rse_id,
parameters=rse,
session=session)
new_rses.append(rse_id)
# Protocols
new_protocols = rse.get('protocols')
if new_protocols:
# update existing, add missing and remove left over protocols
old_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in rse_module.get_rse_protocols(
rse_id=rse_id, session=session)['protocols']]
missing_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} not in old_protocols
]
outdated_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} in old_protocols
]
new_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in new_protocols]
to_be_removed_protocols = [
old_protocol for old_protocol in old_protocols
if old_protocol not in new_protocols
]
for protocol in outdated_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse_id=rse_id,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
for protocol in missing_protocols:
rse_module.add_protocol(rse_id=rse_id,
parameter=protocol,
session=session)
for protocol in to_be_removed_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
rse_module.del_protocols(rse_id=rse_id,
scheme=scheme,
port=port,
hostname=hostname,
session=session)
# Limits
old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']:
limit = rse.get(limit_name)
if limit:
if limit_name in old_limits:
rse_module.delete_rse_limit(rse_id=rse_id,
name=limit_name,
session=session)
rse_module.set_rse_limits(rse_id=rse_id,
name=limit_name,
value=limit,
session=session)
# Attributes
attributes = rse.get('attributes', {})
attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm')
attributes['verify_checksum'] = rse.get('verify_checksum')
old_attributes = rse_module.list_rse_attributes(rse_id=rse_id,
session=session)
for attr in attributes:
value = attributes[attr]
if value is not None:
if attr in old_attributes:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
# set deleted flag to RSEs that are missing in the import data
old_rses = [
old_rse['id'] for old_rse in rse_module.list_rses(session=session)
]
for old_rse in old_rses:
if old_rse not in new_rses:
try:
rse_module.del_rse(rse_id=old_rse, session=session)
except RSEOperationNotSupported:
pass
def __init__(self,
account,
rses,
weight,
copies,
ignore_account_limit=False,
session=None):
"""
Initialize the RSE Selector.
:param account: Account owning the rule.
:param rses: List of rse dictionaries.
:param weight: Weighting to use.
:param copies: Number of copies to create.
:param ignore_account_limit: Flag if the quota should be ignored.
:param session: DB Session in use.
:raises: InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
"""
self.account = account
self.rses = [] # [{'rse_id':, 'weight':, 'staging_area'}]
self.copies = copies
if weight is not None:
for rse in rses:
attributes = list_rse_attributes(rse_id=rse['id'],
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
if weight not in attributes:
continue # The RSE does not have the required weight set, therefore it is ignored
try:
self.rses.append({
'rse_id': rse['id'],
'weight': float(attributes[weight]),
'mock_rse': attributes.get('mock', False),
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
except ValueError:
raise InvalidRuleWeight(
'The RSE \'%s\' has a non-number specified for the weight \'%s\''
% (rse['rse'], weight))
else:
for rse in rses:
mock_rse = has_rse_attribute(rse['id'],
'mock',
session=session)
availability_write = True if rse.get('availability',
7) & 2 else False
self.rses.append({
'rse_id': rse['id'],
'weight': 1,
'mock_rse': mock_rse,
'availability_write': availability_write,
'staging_area': rse['staging_area']
})
if len(self.rses) < self.copies:
raise InsufficientTargetRSEs(
'Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)'
% (self.copies, len(self.rses)))
if has_account_attribute(account=account, key='admin',
session=session) or ignore_account_limit:
for rse in self.rses:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
else:
for rse in self.rses:
if rse['mock_rse']:
rse['quota_left'] = float('inf')
rse['space_left'] = float('inf')
else:
quota_limit = get_account_limit(account=account,
rse_id=rse['rse_id'],
session=session)
if quota_limit is None:
rse['quota_left'] = 0
else:
rse['quota_left'] = quota_limit - get_usage(
rse_id=rse['rse_id'],
account=account,
session=session)['bytes']
space_limit = get_rse_limits(
name='MaxSpaceAvailable',
rse_id=rse['rse_id'],
session=session).get('MaxSpaceAvailable')
if space_limit is None or space_limit < 0:
rse['space_left'] = float('inf')
else:
rse['space_left'] = space_limit - get_rse_counter(
rse_id=rse['rse_id'], session=session)['bytes']
self.rses = [rse for rse in self.rses if rse['quota_left'] > 0]
if len(self.rses) < self.copies:
raise InsufficientAccountLimit(
'There is insufficient quota on any of the target RSE\'s to fullfill the operation.'
)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {
'X-Rucio-Type': 'user',
'X-Rucio-Account': 'root',
'X-Rucio-Auth-Token': str(token)
}
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already existed before
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data3))
assert_equal(r2.status, 201)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=self.data1)
# RSE that had not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already exists
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def __check_rse_usage(rse: str,
rse_id: str,
greedy: bool = False,
logger: 'Callable' = logging.log) -> 'Tuple[int, bool]':
"""
Internal method to check RSE usage and limits.
:param rse: The RSE name.
:param rse_id: The RSE id.
:param greedy: If True, needed_free_space will be set to 1TB regardless of actual rse usage.
:returns: needed_free_space, only_delete_obsolete.
"""
result = REGION.get('rse_usage_%s' % rse_id)
if result is NO_VALUE:
needed_free_space, used, free, obsolete = 0, 0, 0, 0
# First of all check if greedy mode is enabled for this RSE or generally
attributes = list_rse_attributes(rse_id=rse_id)
rse_attr_greedy = attributes.get('greedyDeletion', False)
if greedy or rse_attr_greedy:
result = (1000000000000, False)
REGION.set('rse_usage_%s' % rse_id, result)
return result
# Get RSE limits
limits = get_rse_limits(rse_id=rse_id)
min_free_space = limits.get('MinFreeSpace', 0)
# Check from which sources to get used and total spaces
# Default is storage
source_for_total_space = attributes.get('source_for_total_space',
'storage')
source_for_used_space = attributes.get('source_for_used_space',
'storage')
logger(
logging.DEBUG,
'RSE: %s, source_for_total_space: %s, source_for_used_space: %s',
rse, source_for_total_space, source_for_used_space)
# Get total, used and obsolete space
rse_usage = get_rse_usage(rse_id=rse_id)
usage = [entry for entry in rse_usage if entry['source'] == 'obsolete']
for var in usage:
obsolete = var['used']
break
usage = [
entry for entry in rse_usage
if entry['source'] == source_for_total_space
]
# If no information is available about disk space, do nothing except if there are replicas with Epoch tombstone
if not usage:
if not obsolete:
result = (needed_free_space, False)
REGION.set('rse_usage_%s' % rse_id, result)
return result
result = (obsolete, True)
REGION.set('rse_usage_%s' % rse_id, result)
return result
# Extract the total and used space
for var in usage:
total, used = var['total'], var['used']
break
if source_for_total_space != source_for_used_space:
usage = [
entry for entry in rse_usage
if entry['source'] == source_for_used_space
]
if not usage:
result = (needed_free_space, False)
REGION.set('rse_usage_%s' % rse_id, result)
return result
for var in usage:
used = var['used']
break
free = total - used
if min_free_space:
needed_free_space = min_free_space - free
# If needed_free_space negative, nothing to delete except if some Epoch tombstoned replicas
if needed_free_space <= 0:
result = (obsolete, True)
else:
result = (needed_free_space, False)
REGION.set('rse_usage_%s' % rse_id, result)
return result
return result
def import_rses(rses,
rse_sync_method='edit',
attr_sync_method='edit',
protocol_sync_method='edit',
vo='def',
session=None):
new_rses = []
for rse_name in rses:
rse = rses[rse_name]
if isinstance(rse.get('rse_type'), string_types):
rse['rse_type'] = RSEType(rse['rse_type'])
if rse_module.rse_exists(rse_name,
vo=vo,
include_deleted=False,
session=session):
# RSE exists and is active
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
session=session)
selected_rse_properties = {
key: rse[key]
for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES
}
rse_module.update_rse(rse_id=rse_id,
parameters=selected_rse_properties,
session=session)
elif rse_module.rse_exists(rse_name,
vo=vo,
include_deleted=True,
session=session):
# RSE exists but in deleted state
# Should only modify the RSE if importer is configured for edit or hard sync
if rse_sync_method in ['edit', 'hard']:
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
include_deleted=True,
session=session)
rse_module.restore_rse(rse_id, session=session)
selected_rse_properties = {
key: rse[key]
for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES
}
rse_module.update_rse(rse_id=rse_id,
parameters=selected_rse_properties,
session=session)
else:
# Config is in RSE append only mode, should not modify the disabled RSE
continue
else:
rse_id = rse_module.add_rse(rse=rse_name,
vo=vo,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
new_rses.append(rse_id)
# Protocols
new_protocols = rse.get('protocols')
if new_protocols:
# update existing, add missing and remove left over protocols
old_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in rse_module.get_rse_protocols(
rse_id=rse_id, session=session)['protocols']]
missing_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} not in old_protocols
]
outdated_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} in old_protocols
]
new_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in new_protocols]
to_be_removed_protocols = [
old_protocol for old_protocol in old_protocols
if old_protocol not in new_protocols
]
if protocol_sync_method == 'append':
outdated_protocols = []
for protocol in outdated_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse_id=rse_id,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
for protocol in missing_protocols:
rse_module.add_protocol(rse_id=rse_id,
parameter=protocol,
session=session)
if protocol_sync_method == 'hard':
for protocol in to_be_removed_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
rse_module.del_protocols(rse_id=rse_id,
scheme=scheme,
port=port,
hostname=hostname,
session=session)
# Limits
old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']:
limit = rse.get(limit_name)
if limit:
if limit_name in old_limits:
rse_module.delete_rse_limits(rse_id=rse_id,
name=limit_name,
session=session)
rse_module.set_rse_limits(rse_id=rse_id,
name=limit_name,
value=limit,
session=session)
# Attributes
attributes = rse.get('attributes', {})
attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm')
attributes['verify_checksum'] = rse.get('verify_checksum')
old_attributes = rse_module.list_rse_attributes(rse_id=rse_id,
session=session)
missing_attributes = [
attribute for attribute in old_attributes
if attribute not in attributes
]
for attr in attributes:
value = attributes[attr]
if value is not None:
if attr in old_attributes:
if attr_sync_method not in ['append']:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
else:
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
if attr_sync_method == 'hard':
for attr in missing_attributes:
if attr != rse_name:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
# set deleted flag to RSEs that are missing in the import data
old_rses = [
old_rse['id'] for old_rse in rse_module.list_rses(session=session)
]
if rse_sync_method == 'hard':
for old_rse in old_rses:
if old_rse not in new_rses:
try:
rse_module.del_rse(rse_id=old_rse, session=session)
except RSEOperationNotSupported:
pass