def check_protocols(rse, test_data):
rse_id = get_rse_id(rse=rse)
protocols = get_rse_protocols(rse_id)
assert_equal(
test_data[rse]['lfn2pfn_algorithm'],
get_rse_attribute('lfn2pfn_algorithm', rse_id=rse_id,
use_cache=False)[0])
assert_equal(
test_data[rse]['verify_checksum'],
get_rse_attribute('verify_checksum', rse_id=rse_id,
use_cache=False)[0])
assert_equal(test_data[rse]['availability_write'],
protocols['availability_write'])
assert_equal(test_data[rse]['availability_read'],
protocols['availability_read'])
assert_equal(test_data[rse]['availability_delete'],
protocols['availability_delete'])
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in protocols['protocols']]
for protocol in test_data[rse]['protocols']:
assert_true({
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol.get('impl', ''),
'prefix': protocol.get('prefix', '')
} in protocols)
def get_rses_to_hostname_mapping():
"""
Return a dictionaries mapping the RSEs to the hostname of the SE
:returns: Dictionary with RSE_id as key and (hostname, rse_info) as value
"""
result = REGION.get('rse_hostname_mapping')
if result is NO_VALUE:
result = {}
all_rses = list_rses()
for rse in all_rses:
try:
rse_protocol = get_rse_protocols(rse_id=rse['id'])
except RSENotFound:
logging.log(
logging.WARNING,
'RSE deleted while constructing rse-to-hostname mapping. Skipping %s',
rse['rse'])
continue
for prot in rse_protocol['protocols']:
if prot['domains']['wan']['delete'] == 1:
result[rse['id']] = (prot['hostname'], rse_protocol)
if rse['id'] not in result:
logging.log(logging.WARNING,
'No default delete protocol for %s', rse['rse'])
REGION.set('rse_hostname_mapping', result)
return result
return result
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
if rse_id is None:
# This can be called directly by client tools if they're co-located on a server
# i.e. running rucio cli on a server and during the test suite.
# We have to map to VO name here for this situations, despite this nominally
# not being a client interface.
rse_id = get_rse_id(rse=rse, vo=map_vo(vo))
return get_rse_protocols(rse_id=rse_id, session=session)
def test_importer_core(self):
""" IMPORTER (CORE): test import. """
import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
# one protocol should be created, one should be updated
check_protocols(self.old_rse_1, self.data1['rses'])
# one protocol should be removed as it is not specified in the import data
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
# RSE 4 should be flagged as deleted as it is missing in the import data
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_data(data=self.data2)
import_data(data=self.data3)
def get_rse_protocols(rse, issuer):
"""
Returns all matching protocols (including detailed information) for the given RSE.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: A dict with all supported protocols and their attibutes.
"""
return rse_module.get_rse_protocols(rse)
def get_rse_protocols(rse, issuer):
"""
Returns all matching protocols (including detailed information) for the given RSE.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: A dict with all supported protocols and their attibutes.
"""
return rse_module.get_rse_protocols(rse)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=deepcopy(self.data1))
# RSE that had not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
new_rse_id = get_rse_id(rse=self.new_rse)
protocols = get_rse_protocols(self.old_rse_id_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in protocols['protocols']]
assert_true({
'hostename': 'hostname3',
'port': 1000,
'scheme': 'scheme3'
} not in protocols)
attributes = list_rse_attributes(rse_id=new_rse_id)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse_id=new_rse_id)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already exists
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse_id=self.old_rse_id_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse_id=self.old_rse_id_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
self.check_accounts(self.data1['accounts'])
with assert_raises(RSENotFound):
get_rse(rse_id=self.old_rse_id_4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def get_rse(rse):
"""
Provides details about the specified RSE.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: a dict with details about the RSE
:raises RSENotFound: if the referred RSE was ot found in the database
"""
return rse_module.get_rse_protocols(rse)
def get_rse_protocols(rse, issuer, vo='def'):
"""
Returns all matching protocols (including detailed information) for the given RSE.
:param rse: The RSE name.
:param issuer: The issuer account.
:param vo: The VO to act on.
:returns: A dict with all supported protocols and their attibutes.
"""
rse_id = rse_module.get_rse_id(rse=rse, vo=vo)
return rse_module.get_rse_protocols(rse_id)
def get_rse(rse):
"""
Provides details about the specified RSE.
:param rse: The RSE name.
:param issuer: The issuer account.
:returns: a dict with details about the RSE
:raises RSENotFound: if the referred RSE was not found in the database
"""
return rse_module.get_rse_protocols(rse)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)}
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
check_rse(self.new_rse, self.data1['rses'])
check_protocols(self.new_rse, self.data1['rses'])
protocols = get_rse_protocols(self.old_rse_1)
protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']]
assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['MinFreeSpace'], 20000)
# RSE 1 that already existed before
check_rse(self.old_rse_1, self.data1['rses'])
check_protocols(self.old_rse_1, self.data1['rses'])
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['MaxBeingDeletedFiles'], 1000)
assert_equal(limits['MinFreeSpace'], 10000)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
with assert_raises(RSENotFound):
get_rse(self.old_rse_3)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3))
assert_equal(r2.status, 201)
def get_rse(rse, vo='def'):
"""
Provides details about the specified RSE.
:param rse: The RSE name.
:param vo: The VO to act on.
:returns: a dict with details about the RSE
:raises RSENotFound: if the referred RSE was not found in the database
"""
rse_id = rse_module.get_rse_id(rse=rse, vo=vo)
return rse_module.get_rse_protocols(rse_id=rse_id)
def get_authfile_prefixes(self):
session = get_session()
session.connection()
prefixes = set()
for rse in list_rses():
rse_id = rse['id']
protocols = get_rse_protocols(rse_id)
if 'protocols' in protocols:
for protocol in protocols['protocols']:
scheme = protocol['scheme']
hostname = protocol['hostname']
port = protocol['port']
prefix = self.clean_prefix_path(protocol['prefix'])
authfile_prefix = f"/{scheme}:/{hostname}:{port}/{prefix}"
prefixes.add(authfile_prefix)
return list(prefixes)
def check_protocols(rse, test_data):
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
vo = {'vo': 'tst'}
else:
vo = {}
rse_id = get_rse_id(rse=rse, **vo)
protocols = get_rse_protocols(rse_id)
assert_equal(
test_data[rse]['lfn2pfn_algorithm'],
get_rse_attribute('lfn2pfn_algorithm', rse_id=rse_id,
use_cache=False)[0])
assert_equal(
test_data[rse]['verify_checksum'],
get_rse_attribute('verify_checksum', rse_id=rse_id,
use_cache=False)[0])
assert_equal(test_data[rse]['availability_write'],
protocols['availability_write'])
assert_equal(test_data[rse]['availability_read'],
protocols['availability_read'])
assert_equal(test_data[rse]['availability_delete'],
protocols['availability_delete'])
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in protocols['protocols']]
for protocol in test_data[rse]['protocols']:
assert_true({
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol.get('impl', ''),
'prefix': protocol.get('prefix', '')
} in protocols)
def get_rses_to_hostname_mapping():
"""
Return a dictionaries mapping the RSEs to the hostname of the SE
:returns: Dictionary with RSE_id as key and (hostname, rse_info) as value
"""
result = REGION.get('rse_hostname_mapping')
if result is NO_VALUE:
result = {}
all_rses = list_rses()
for rse in all_rses:
rse_protocol = get_rse_protocols(rse_id=rse['id'])
for prot in rse_protocol['protocols']:
if prot['domains']['wan']['delete'] == 1:
result[rse['id']] = (prot['hostname'], rse_protocol)
if rse['id'] not in result:
logging.warn('No default delete protocol for %s', rse['rse'])
REGION.set('rse_hostname_mapping', result)
return result
return result
def reaper(rses, worker_number=0, child_number=0, total_children=1, chunk_size=100,
once=False, greedy=False, scheme=None, delay_seconds=0):
"""
Main loop to select and delete files.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param worker_number: The worker number.
:param child_number: The child number.
:param total_children: The total number of children created per worker.
:param chunk_size: the size of chunk for deletion.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param scheme: Force the reaper to use a particular protocol, e.g., mock.
"""
logging.info('Starting Reaper: Worker %(worker_number)s, '
'child %(child_number)s will work on RSEs: ' % locals() + ', '.join([rse['rse'] for rse in rses]))
pid = os.getpid()
thread = threading.current_thread()
hostname = socket.gethostname()
executable = ' '.join(sys.argv)
# Generate a hash just for the subset of RSEs
rse_names = [rse['rse'] for rse in rses]
hash_executable = hashlib.sha256((sys.argv[0] + ''.join(rse_names)).encode()).hexdigest()
sanity_check(executable=None, hostname=hostname)
nothing_to_do = {}
while not GRACEFUL_STOP.is_set():
try:
# heartbeat
heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable)
checkpoint_time = datetime.datetime.now()
# logging.info('Reaper({0[worker_number]}/{0[child_number]}): Live gives {0[heartbeat]}'.format(locals()))
max_deleting_rate = 0
for rse in sort_rses(rses):
try:
if checkpoint_time + datetime.timedelta(minutes=1) < datetime.datetime.now():
heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable)
# logging.info('Reaper({0[worker_number]}/{0[child_number]}): Live gives {0[heartbeat]}'.format(locals()))
checkpoint_time = datetime.datetime.now()
if rse['id'] in nothing_to_do and nothing_to_do[rse['id']] > datetime.datetime.now():
continue
logging.info('Reaper %s-%s: Running on RSE %s %s', worker_number, child_number,
rse['rse'], nothing_to_do.get(rse['id']))
rse_info = rsemgr.get_rse_info(rse_id=rse['id'])
rse_protocol = rse_core.get_rse_protocols(rse_id=rse['id'])
if not rse_protocol['availability_delete']:
logging.info('Reaper %s-%s: RSE %s is not available for deletion', worker_number, child_number, rse_info['rse'])
nothing_to_do[rse['id']] = datetime.datetime.now() + datetime.timedelta(minutes=30)
continue
# Temporary hack to force gfal for deletion
for protocol in rse_info['protocols']:
if protocol['impl'] == 'rucio.rse.protocols.srm.Default' or protocol['impl'] == 'rucio.rse.protocols.gsiftp.Default':
protocol['impl'] = 'rucio.rse.protocols.gfal.Default'
needed_free_space, max_being_deleted_files = None, 100
needed_free_space_per_child = None
if not greedy:
max_being_deleted_files, needed_free_space, used, free = __check_rse_usage(rse_id=rse['id'])
logging.info('Reaper %(worker_number)s-%(child_number)s: Space usage for RSE %(rse)s - max_being_deleted_files: %(max_being_deleted_files)s, needed_free_space: %(needed_free_space)s, used: %(used)s, free: %(free)s' % locals())
if needed_free_space <= 0:
needed_free_space, needed_free_space_per_child = 0, 0
logging.info('Reaper %s-%s: free space is above minimum limit for %s', worker_number, child_number, rse['rse'])
else:
if total_children and total_children > 0:
needed_free_space_per_child = needed_free_space / float(total_children)
start = time.time()
with monitor.record_timer_block('reaper.list_unlocked_replicas'):
replicas = list_unlocked_replicas(rse_id=rse['id'],
bytes=needed_free_space_per_child,
limit=max_being_deleted_files,
worker_number=child_number,
total_workers=total_children,
delay_seconds=delay_seconds)
logging.debug('Reaper %s-%s: list_unlocked_replicas on %s for %s bytes in %s seconds: %s replicas', worker_number, child_number, rse['rse'], needed_free_space_per_child, time.time() - start, len(replicas))
if not replicas:
nothing_to_do[rse['id']] = datetime.datetime.now() + datetime.timedelta(minutes=30)
logging.info('Reaper %s-%s: No replicas to delete %s. The next check will occur at %s',
worker_number, child_number, rse['rse'],
nothing_to_do[rse['id']])
continue
prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme)
for files in chunks(replicas, chunk_size):
logging.debug('Reaper %s-%s: Running on : %s', worker_number, child_number, str(files))
try:
update_replicas_states(replicas=[dict(list(replica.items()) + [('state', ReplicaState.BEING_DELETED), ('rse_id', rse['id'])]) for replica in files], nowait=True)
for replica in files:
try:
replica['pfn'] = str(list(rsemgr.lfns2pfns(rse_settings=rse_info,
lfns=[{'scope': replica['scope'].external, 'name': replica['name'], 'path': replica['path']}],
operation='delete', scheme=scheme).values())[0])
except (ReplicaUnAvailable, ReplicaNotFound) as error:
err_msg = 'Failed to get pfn UNAVAILABLE replica %s:%s on %s with error %s' % (replica['scope'], replica['name'], rse['rse'], str(error))
logging.warning('Reaper %s-%s: %s', worker_number, child_number, err_msg)
replica['pfn'] = None
monitor.record_counter(counters='reaper.deletion.being_deleted', delta=len(files))
try:
deleted_files = []
prot.connect()
for replica in files:
try:
deletion_dict = {'scope': replica['scope'].external,
'name': replica['name'],
'rse': rse_info['rse'],
'rse_id': rse_info['id'],
'file-size': replica['bytes'],
'bytes': replica['bytes'],
'url': replica['pfn'],
'protocol': prot.attributes['scheme']}
if replica['scope'].vo != 'def':
deletion_dict['vo'] = replica['scope'].vo
logging.info('Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])
start = time.time()
if rse['staging_area'] or rse['rse'].endswith("STAGING"):
logging.warning('Reaper %s-%s: Deletion STAGING of %s:%s as %s on %s, will only delete the catalog and not do physical deletion',
worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])
else:
if replica['pfn']:
pfn = replica['pfn']
# sign the URL if necessary
if prot.attributes['scheme'] == 'https' and rse_info['sign_url'] is not None:
pfn = get_signed_url(rse['id'], rse_info['sign_url'], 'delete', pfn)
prot.delete(pfn)
else:
logging.warning('Reaper %s-%s: Deletion UNAVAILABLE of %s:%s as %s on %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])
monitor.record_timer('daemons.reaper.delete.%s.%s' % (prot.attributes['scheme'], rse['rse']), (time.time() - start) * 1000)
duration = time.time() - start
deleted_files.append({'scope': replica['scope'], 'name': replica['name']})
deletion_dict['duration'] = duration
add_message('deletion-done', deletion_dict)
logging.info('Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], duration)
except SourceNotFound:
err_msg = 'Deletion NOTFOUND of %s:%s as %s on %s' % (replica['scope'], replica['name'], replica['pfn'], rse['rse'])
logging.warning(err_msg)
deleted_files.append({'scope': replica['scope'], 'name': replica['name']})
if replica['state'] == ReplicaState.AVAILABLE:
deletion_dict['reason'] = str(err_msg)
add_message('deletion-failed', deletion_dict)
except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error:
logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(error))
deletion_dict['reason'] = str(error)
add_message('deletion-failed', deletion_dict)
except Exception as error:
logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc()))
deletion_dict['reason'] = str(error)
add_message('deletion-failed', deletion_dict)
except:
logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc()))
except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error:
for replica in files:
logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(error))
payload = {'scope': replica['scope'].external,
'name': replica['name'],
'rse': rse_info['rse'],
'rse_id': rse_info['id'],
'file-size': replica['bytes'],
'bytes': replica['bytes'],
'url': replica['pfn'],
'reason': str(error),
'protocol': prot.attributes['scheme']}
if replica['scope'].vo != 'def':
deletion_dict['vo'] = replica['scope'].vo
add_message('deletion-failed', payload)
break
finally:
prot.close()
start = time.time()
with monitor.record_timer_block('reaper.delete_replicas'):
delete_replicas(rse_id=rse['id'], files=deleted_files)
logging.debug('Reaper %s-%s: delete_replicas successes %s %s %s', worker_number, child_number, rse['rse'], len(deleted_files), time.time() - start)
monitor.record_counter(counters='reaper.deletion.done', delta=len(deleted_files))
except DatabaseException as error:
logging.warning('Reaper %s-%s: DatabaseException %s', worker_number, child_number, str(error))
except UnsupportedOperation as error:
logging.warning('Reaper %s-%s: UnsupportedOperation %s', worker_number, child_number, str(error))
except:
logging.critical(traceback.format_exc())
except RSENotFound as error:
logging.warning('Reaper %s-%s: RSE not found %s', worker_number, child_number, str(error))
except:
logging.critical(traceback.format_exc())
if once:
break
time.sleep(1)
except DatabaseException as error:
logging.warning('Reaper: %s', str(error))
except:
logging.critical(traceback.format_exc())
die(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable)
logging.info('Graceful stop requested')
logging.info('Graceful stop done')
return
def tmp_rse_info(rse, session=None):
rse_id = get_rse_id(rse=rse)
return get_rse_protocols(rse_id=rse_id, session=session)
def import_data(data, session=None):
"""
Import data to add and update records in Rucio.
:param data: data to be imported as dictionary.
:param session: database session in use.
"""
# RSEs
rses = data.get('rses')
if rses:
for rse in rses:
protocols = rse.get('protocols')
if protocols:
protocols = protocols.get('protocols')
del rse['protocols']
rse_name = rse['rse']
del rse['rse']
if not rse_module.rse_exists(rse_name, session=session):
rse_module.add_rse(rse_name,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_name, rse, session=session)
# Protocols
if protocols:
old_protocols = rse_module.get_rse_protocols(rse=rse_name,
session=session)
for protocol in protocols:
scheme = protocol.get('scheme')
hostname = protocol.get('hostname')
port = protocol.get('port')
intersection = [
old_protocol
for old_protocol in old_protocols['protocols']
if old_protocol['scheme'] == scheme
and old_protocol['hostname'] == hostname
and old_protocol['port'] == port
]
if intersection:
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse=rse_name,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
else:
rse_module.add_protocol(rse=rse_name,
parameter=protocol,
session=session)
# Limits
limits = rse.get('limits')
if limits:
old_limits = rse_module.get_rse_limits(rse=rse_name,
session=session)
for limit in limits:
if limit in old_limits:
rse_module.delete_rse_limit(rse=rse_name,
name=limit,
session=session)
rse_module.set_rse_limits(rse=rse_name,
name=limit,
value=limits[limit],
session=session)
# Transfer limits
transfer_limits = rse.get('transfer_limits')
if transfer_limits:
for limit in transfer_limits:
old_transfer_limits = rse_module.get_rse_transfer_limits(
rse=rse_name, activity=limit, session=session)
if limit in old_transfer_limits:
rse_module.delete_rse_transfer_limits(rse=rse_name,
activity=limit,
session=session)
max_transfers = transfer_limits[limit].items(
)[0][1]['max_transfers']
rse_module.set_rse_transfer_limits(
rse=rse_name,
activity=limit,
max_transfers=max_transfers,
session=session)
# Attributes
attributes = rse.get('attributes')
if attributes:
old_attributes = rse_module.list_rse_attributes(
rse=rse_name, session=session)
for attr in attributes:
if attr in old_attributes:
rse_module.del_rse_attribute(rse=rse_name,
key=attr,
session=session)
rse_module.add_rse_attribute(rse=rse_name,
key=attr,
value=attributes[attr],
session=session)
# Distances
distances = data.get('distances')
if distances:
for src_rse_name in distances:
src = rse_module.get_rse_id(src_rse_name, session=session)
for dest_rse_name in distances[src_rse_name]:
dest = rse_module.get_rse_id(dest_rse_name, session=session)
distance = distances[src_rse_name][dest_rse_name]
del distance['src_rse_id']
del distance['dest_rse_id']
old_distance = distance_module.get_distances(src_rse_id=src,
dest_rse_id=dest,
session=session)
if old_distance:
distance_module.update_distances(src_rse_id=src,
dest_rse_id=dest,
parameters=distance,
session=session)
else:
distance_module.add_distance(
src_rse_id=src,
dest_rse_id=dest,
ranking=distance.get('ranking'),
agis_distance=distance.get('agis_distance'),
geoip_distance=distance.get('geoip_distance'),
active=distance.get('active'),
submitted=distance.get('submitted'),
transfer_speed=distance.get('transfer_speed'),
finished=distance.get('finished'),
failed=distance.get('failed'),
session=session)
def reaper(rses=[],
worker_number=1,
total_workers=1,
chunk_size=100,
once=False,
scheme=None):
"""
Main loop to select and delete files.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param worker_number: The worker number.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param once: If True, only runs one iteration of the main loop.
:param scheme: Force the reaper to use a particular protocol, e.g., mock.
"""
logging.info('Starting Light Reaper %s-%s: Will work on RSEs: %s',
worker_number, total_workers, str(rses))
pid = os.getpid()
thread = threading.current_thread()
hostname = socket.gethostname()
executable = ' '.join(sys.argv)
hash_executable = hashlib.sha256(sys.argv[0] + ''.join(rses)).hexdigest()
sanity_check(executable=None, hostname=hostname)
while not GRACEFUL_STOP.is_set():
try:
# heartbeat
heartbeat = live(executable=executable,
hostname=hostname,
pid=pid,
thread=thread,
hash_executable=hash_executable)
logging.info(
'Light Reaper({0[worker_number]}/{0[total_workers]}): Live gives {0[heartbeat]}'
.format(locals()))
nothing_to_do = True
random.shuffle(rses)
for rse in rses:
replicas = list_expired_temporary_dids(
rse=rse,
limit=chunk_size,
worker_number=worker_number,
total_workers=total_workers)
rse_info = rsemgr.get_rse_info(rse)
rse_protocol = rse_core.get_rse_protocols(rse)
prot = rsemgr.create_protocol(rse_info,
'delete',
scheme=scheme)
deleted_replicas = []
try:
prot.connect()
for replica in replicas:
nothing_to_do = False
try:
# pfn = str(rsemgr.lfns2pfns(rse_settings=rse_info,
# lfns=[{'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']}],
# operation='delete', scheme=scheme).values()[0])
pfn = 's3://%s%s%s' % (prot.attributes['hostname'],
prot.attributes['prefix'],
replica['name'])
# logging.debug('Light Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse)
start = time.time()
prot.delete(pfn)
duration = time.time() - start
logging.info(
'Light Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds',
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse, duration)
add_message(
'deletion-done', {
'scope': replica['scope'],
'name': replica['name'],
'rse': rse,
'file-size': replica.get('bytes') or 0,
'bytes': replica.get('bytes') or 0,
'url': pfn,
'duration': duration
})
deleted_replicas.append(replica)
except SourceNotFound:
err_msg = 'Light Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % (
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse)
logging.warning(err_msg)
deleted_replicas.append(replica)
except (ServiceUnavailable, RSEAccessDenied,
ResourceTemporaryUnavailable) as error:
err_msg = 'Light Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse, str(error))
logging.warning(err_msg)
add_message(
'deletion-failed', {
'scope': replica['scope'],
'name': replica['name'],
'rse': rse,
'file-size': replica['bytes'] or 0,
'bytes': replica['bytes'] or 0,
'url': pfn,
'reason': str(error)
})
except:
logging.critical(traceback.format_exc())
finally:
prot.close()
delete_temporary_dids(dids=deleted_replicas)
if once:
break
if once:
break
if nothing_to_do:
logging.info(
'Light Reaper %s-%s: Nothing to do. I will sleep for 60s',
worker_number, total_workers)
time.sleep(60)
except DatabaseException, error:
logging.warning('Reaper: %s', str(error))
except:
def import_rses(rses, vo='def', session=None):
new_rses = []
for rse_name in rses:
rse = rses[rse_name]
if isinstance(rse.get('rse_type'), string_types):
rse['rse_type'] = RSEType.from_string(str(rse['rse_type']))
try:
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
session=session)
except RSENotFound:
rse_id = rse_module.add_rse(rse=rse_name,
vo=vo,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
else:
rse_module.update_rse(rse_id=rse_id,
parameters=rse,
session=session)
new_rses.append(rse_id)
# Protocols
new_protocols = rse.get('protocols')
if new_protocols:
# update existing, add missing and remove left over protocols
old_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in rse_module.get_rse_protocols(
rse_id=rse_id, session=session)['protocols']]
missing_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} not in old_protocols
]
outdated_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} in old_protocols
]
new_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in new_protocols]
to_be_removed_protocols = [
old_protocol for old_protocol in old_protocols
if old_protocol not in new_protocols
]
for protocol in outdated_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse_id=rse_id,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
for protocol in missing_protocols:
rse_module.add_protocol(rse_id=rse_id,
parameter=protocol,
session=session)
for protocol in to_be_removed_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
rse_module.del_protocols(rse_id=rse_id,
scheme=scheme,
port=port,
hostname=hostname,
session=session)
# Limits
old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']:
limit = rse.get(limit_name)
if limit:
if limit_name in old_limits:
rse_module.delete_rse_limit(rse_id=rse_id,
name=limit_name,
session=session)
rse_module.set_rse_limits(rse_id=rse_id,
name=limit_name,
value=limit,
session=session)
# Attributes
attributes = rse.get('attributes', {})
attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm')
attributes['verify_checksum'] = rse.get('verify_checksum')
old_attributes = rse_module.list_rse_attributes(rse_id=rse_id,
session=session)
for attr in attributes:
value = attributes[attr]
if value is not None:
if attr in old_attributes:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
# set deleted flag to RSEs that are missing in the import data
old_rses = [
old_rse['id'] for old_rse in rse_module.list_rses(session=session)
]
for old_rse in old_rses:
if old_rse not in new_rses:
try:
rse_module.del_rse(rse_id=old_rse, session=session)
except RSEOperationNotSupported:
pass
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
if rse_id is None:
rse_id = get_rse_id(rse=rse, vo=vo)
return get_rse_protocols(rse_id=rse_id, session=session)
from dogpile.cache import make_region
from rucio.core.rse import list_rses, get_rse_protocols
rse_region = make_region().configure(
'dogpile.cache.memcached', # 'dogpile.cache.memory'
expiration_time=1,
arguments={'url': "127.0.0.1:11211", 'distributed_lock': True},
)
if __name__ == '__main__':
for rse in list_rses():
rse_info = get_rse_protocols(rse['rse'])
print rse['rse'], rse_info
rse_region.set(str(rse['rse']), rse_info)
# rse_region.delete(str(rse['rse']))
def import_rses(rses,
rse_sync_method='edit',
attr_sync_method='edit',
protocol_sync_method='edit',
vo='def',
session=None):
new_rses = []
for rse_name in rses:
rse = rses[rse_name]
if isinstance(rse.get('rse_type'), string_types):
rse['rse_type'] = RSEType(rse['rse_type'])
if rse_module.rse_exists(rse_name,
vo=vo,
include_deleted=False,
session=session):
# RSE exists and is active
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
session=session)
selected_rse_properties = {
key: rse[key]
for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES
}
rse_module.update_rse(rse_id=rse_id,
parameters=selected_rse_properties,
session=session)
elif rse_module.rse_exists(rse_name,
vo=vo,
include_deleted=True,
session=session):
# RSE exists but in deleted state
# Should only modify the RSE if importer is configured for edit or hard sync
if rse_sync_method in ['edit', 'hard']:
rse_id = rse_module.get_rse_id(rse=rse_name,
vo=vo,
include_deleted=True,
session=session)
rse_module.restore_rse(rse_id, session=session)
selected_rse_properties = {
key: rse[key]
for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES
}
rse_module.update_rse(rse_id=rse_id,
parameters=selected_rse_properties,
session=session)
else:
# Config is in RSE append only mode, should not modify the disabled RSE
continue
else:
rse_id = rse_module.add_rse(rse=rse_name,
vo=vo,
deterministic=rse.get('deterministic'),
volatile=rse.get('volatile'),
city=rse.get('city'),
region_code=rse.get('region_code'),
country_name=rse.get('country_name'),
staging_area=rse.get('staging_area'),
continent=rse.get('continent'),
time_zone=rse.get('time_zone'),
ISP=rse.get('ISP'),
rse_type=rse.get('rse_type'),
latitude=rse.get('latitude'),
longitude=rse.get('longitude'),
ASN=rse.get('ASN'),
availability=rse.get('availability'),
session=session)
new_rses.append(rse_id)
# Protocols
new_protocols = rse.get('protocols')
if new_protocols:
# update existing, add missing and remove left over protocols
old_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in rse_module.get_rse_protocols(
rse_id=rse_id, session=session)['protocols']]
missing_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} not in old_protocols
]
outdated_protocols = [
new_protocol for new_protocol in new_protocols if {
'scheme': new_protocol['scheme'],
'hostname': new_protocol['hostname'],
'port': new_protocol['port']
} in old_protocols
]
new_protocols = [{
'scheme': protocol['scheme'],
'hostname': protocol['hostname'],
'port': protocol['port']
} for protocol in new_protocols]
to_be_removed_protocols = [
old_protocol for old_protocol in old_protocols
if old_protocol not in new_protocols
]
if protocol_sync_method == 'append':
outdated_protocols = []
for protocol in outdated_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
del protocol['scheme']
del protocol['hostname']
del protocol['port']
rse_module.update_protocols(rse_id=rse_id,
scheme=scheme,
data=protocol,
hostname=hostname,
port=port,
session=session)
for protocol in missing_protocols:
rse_module.add_protocol(rse_id=rse_id,
parameter=protocol,
session=session)
if protocol_sync_method == 'hard':
for protocol in to_be_removed_protocols:
scheme = protocol['scheme']
port = protocol['port']
hostname = protocol['hostname']
rse_module.del_protocols(rse_id=rse_id,
scheme=scheme,
port=port,
hostname=hostname,
session=session)
# Limits
old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session)
for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']:
limit = rse.get(limit_name)
if limit:
if limit_name in old_limits:
rse_module.delete_rse_limits(rse_id=rse_id,
name=limit_name,
session=session)
rse_module.set_rse_limits(rse_id=rse_id,
name=limit_name,
value=limit,
session=session)
# Attributes
attributes = rse.get('attributes', {})
attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm')
attributes['verify_checksum'] = rse.get('verify_checksum')
old_attributes = rse_module.list_rse_attributes(rse_id=rse_id,
session=session)
missing_attributes = [
attribute for attribute in old_attributes
if attribute not in attributes
]
for attr in attributes:
value = attributes[attr]
if value is not None:
if attr in old_attributes:
if attr_sync_method not in ['append']:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
else:
rse_module.add_rse_attribute(rse_id=rse_id,
key=attr,
value=value,
session=session)
if attr_sync_method == 'hard':
for attr in missing_attributes:
if attr != rse_name:
rse_module.del_rse_attribute(rse_id=rse_id,
key=attr,
session=session)
# set deleted flag to RSEs that are missing in the import data
old_rses = [
old_rse['id'] for old_rse in rse_module.list_rses(session=session)
]
if rse_sync_method == 'hard':
for old_rse in old_rses:
if old_rse not in new_rses:
try:
rse_module.del_rse(rse_id=old_rse, session=session)
except RSEOperationNotSupported:
pass
def reaper(rses, worker_number=1, child_number=1, total_children=1, chunk_size=100, once=False, greedy=False, scheme=None, exclude_rses=None, delay_seconds=0):
"""
Main loop to select and delete files.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param worker_number: The worker number.
:param child_number: The child number.
:param total_children: The total number of children created per worker.
:param chunk_size: the size of chunk for deletion.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param scheme: Force the reaper to use a particular protocol, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
"""
logging.info('Starting reaper: worker %(worker_number)s, child %(child_number)s' % locals())
while not graceful_stop.is_set():
try:
max_deleting_rate = 0
for rse in rses:
deleting_rate = 0
rse_info = rsemgr.get_rse_info(rse['rse'])
rse_protocol = rse_core.get_rse_protocols(rse['rse'])
if not rse_protocol['availability_delete']:
logging.info('Reaper %s-%s: RSE %s is not available for deletion' % (worker_number, child_number, rse_info['rse']))
continue
# Temporary hack to force gfal for deletion
for protocol in rse_info['protocols']:
if protocol['impl'] == 'rucio.rse.protocols.srm.Default' or protocol['impl'] == 'rucio.rse.protocols.gsiftp.Default':
protocol['impl'] = 'rucio.rse.protocols.gfal.Default'
logging.info('Reaper %s-%s: Running on RSE %s' % (worker_number, child_number, rse_info['rse']))
try:
needed_free_space, max_being_deleted_files = None, 10000
if not greedy:
max_being_deleted_files, needed_free_space, used, free = __check_rse_usage(rse=rse['rse'], rse_id=rse['id'])
logging.info('Reaper %(worker_number)s-%(child_number)s: Space usage for RSE %(rse)s - max_being_deleted_files: %(max_being_deleted_files)s, needed_free_space: %(needed_free_space)s, used: %(used)s, free: %(free)s' % locals())
if needed_free_space <= 0:
needed_free_space = 0
logging.info('Reaper %s-%s: free space is above minimum limit for %s' % (worker_number, child_number, rse['rse']))
s = time.time()
with monitor.record_timer_block('reaper.list_unlocked_replicas'):
replicas = list_unlocked_replicas(rse=rse['rse'], bytes=needed_free_space, limit=max_being_deleted_files, worker_number=child_number, total_workers=total_children, delay_seconds=delay_seconds)
logging.debug('Reaper %s-%s: list_unlocked_replicas %s %s %s' % (worker_number, child_number, rse['rse'], time.time() - s, len(replicas)))
if not replicas:
logging.info('Reaper %s-%s: nothing to do for %s' % (worker_number, child_number, rse['rse']))
continue
p = rsemgr.create_protocol(rse_info, 'delete', scheme=None)
for files in chunks(replicas, chunk_size):
logging.debug('Reaper %s-%s: Running on : %s' % (worker_number, child_number, str(files)))
try:
s = time.time()
update_replicas_states(replicas=[dict(replica.items() + [('state', ReplicaState.BEING_DELETED), ('rse_id', rse['id'])]) for replica in files])
for replica in files:
try:
replica['pfn'] = str(rsemgr.lfns2pfns(rse_settings=rse_info, lfns=[{'scope': replica['scope'], 'name': replica['name']}, ], operation='delete').values()[0])
except ReplicaUnAvailable as e:
err_msg = 'Failed to get pfn UNAVAILABLE replica %s:%s on %s with error %s' % (replica['scope'], replica['name'], rse['rse'], str(e))
logging.warning('Reaper %s-%s: %s' % (worker_number, child_number, err_msg))
replica['pfn'] = None
add_message('deletion-planned', {'scope': replica['scope'],
'name': replica['name'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'rse': rse_info['rse']})
# logging.debug('update_replicas_states %s' % (time.time() - s))
monitor.record_counter(counters='reaper.deletion.being_deleted', delta=len(files))
if not scheme:
try:
deleted_files = []
p.connect()
for replica in files:
try:
logging.info('Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']))
s = time.time()
if rse['staging_area'] or rse['rse'].endswith("STAGING"):
logging.warning('Reaper %s-%s: Deletion STAGING of %s:%s as %s on %s, will only delete the catalog and not do physical deletion' % (worker_number,
child_number,
replica['scope'],
replica['name'],
replica['pfn'],
rse['rse']))
else:
if replica['pfn']:
p.delete(replica['pfn'])
else:
logging.warning('Reaper %s-%s: Deletion UNAVAILABLE of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']))
monitor.record_timer('daemons.reaper.delete.%s.%s' % (p.attributes['scheme'], rse['rse']), (time.time()-s)*1000)
duration = time.time() - s
deleted_files.append({'scope': replica['scope'], 'name': replica['name']})
add_message('deletion-done', {'scope': replica['scope'],
'name': replica['name'],
'rse': rse_info['rse'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'duration': duration})
logging.info('Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']))
except SourceNotFound:
err_msg = 'Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])
logging.warning(err_msg)
deleted_files.append({'scope': replica['scope'], 'name': replica['name']})
add_message('deletion-failed', {'scope': replica['scope'],
'name': replica['name'],
'rse': rse_info['rse'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'reason': err_msg})
except (ServiceUnavailable, RSEAccessDenied) as e:
logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(e)))
add_message('deletion-failed', {'scope': replica['scope'],
'name': replica['name'],
'rse': rse_info['rse'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'reason': str(e)})
except Exception as e:
logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc())))
add_message('deletion-failed', {'scope': replica['scope'],
'name': replica['name'],
'rse': rse_info['rse'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'reason': str(e)})
except:
logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc())))
except (ServiceUnavailable, RSEAccessDenied) as e:
for replica in files:
logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(e)))
add_message('deletion-failed', {'scope': replica['scope'],
'name': replica['name'],
'rse': rse_info['rse'],
'file-size': replica['bytes'],
'url': replica['pfn'],
'reason': str(e)})
finally:
p.close()
s = time.time()
with monitor.record_timer_block('reaper.delete_replicas'):
delete_replicas(rse=rse['rse'], files=deleted_files)
logging.debug('Reaper %s-%s: delete_replicas successes %s %s %s' % (worker_number, child_number, rse['rse'], len(deleted_files), time.time() - s))
monitor.record_counter(counters='reaper.deletion.done', delta=len(deleted_files))
deleting_rate += len(deleted_files)
except:
logging.critical(traceback.format_exc())
deleting_rate = deleting_rate * 1.0 / max_being_deleted_files
if deleting_rate > max_deleting_rate:
max_deleting_rate = deleting_rate
except:
logging.critical(traceback.format_exc())
if once:
break
logging.info(" Reaper %s-%s: max_deleting_rate: %s " % (worker_number, child_number, max_deleting_rate))
sleep_time = int((1 - max_deleting_rate) * 60 + 1)
time.sleep(sleep_time)
except:
logging.critical(traceback.format_exc())
logging.info('Graceful stop requested')
logging.info('Graceful stop done')
def ddmendpoint_preferred_protocol(ddmendpoint):
return next(
p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols']
if p['domains']['wan']['read'] == 1)
def test_importer_client(self):
""" IMPORTER (CLIENT): test import. """
import_client = ImportClient()
import_client.import_data(data=self.data1)
# RSE that had not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already exists
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
import_client.import_data(data=self.data2)
import_client.import_data(data=self.data3)
def test_importer_rest(self):
""" IMPORTER (REST): test import. """
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {
'X-Rucio-Type': 'user',
'X-Rucio-Account': 'root',
'X-Rucio-Auth-Token': str(token)
}
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data1))
assert_equal(r2.status, 201)
# RSE that not existed before
rse = get_rse(self.new_rse)
assert_equal(rse['availability'], 5)
assert_equal(rse['city'], 'NewCity')
assert_equal(rse['rse_type'], RSEType.TAPE)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port']
} for protocol in get_rse_protocols(self.new_rse)['protocols']]
assert_true({
'scheme': 'scheme',
'hostname': 'hostname',
'port': 1000
} in protocols)
attributes = list_rse_attributes(rse=self.new_rse)
assert_equal(attributes['attr1'], 'test')
limits = get_rse_limits(rse=self.new_rse)
assert_equal(limits['limit1'], 0)
transfer_limits = get_rse_transfer_limits(rse=self.new_rse)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.new_rse)]['max_transfers'], 1)
# RSE 1 that already existed before
rse = get_rse(self.old_rse_1)
assert_equal(rse['rse'], self.old_rse_1)
protocols = [{
'hostname': protocol['hostname'],
'scheme': protocol['scheme'],
'port': protocol['port'],
'impl': protocol['impl'],
'prefix': protocol['prefix']
} for protocol in get_rse_protocols(self.old_rse_1)['protocols']]
assert_true({
'scheme': 'scheme1',
'hostname': 'hostname1',
'port': 1000,
'prefix': 'prefix',
'impl': 'impl1'
} in protocols)
assert_true({
'scheme': 'scheme2',
'hostname': 'hostname2',
'port': 1001,
'impl': 'impl',
'prefix': ''
} in protocols)
attributes = list_rse_attributes(rse=self.old_rse_1)
assert_equal(attributes['attr1'], 'test1')
assert_equal(attributes['attr2'], 'test2')
limits = get_rse_limits(rse=self.old_rse_1)
assert_equal(limits['limit1'], 0)
assert_equal(limits['limit2'], 2)
transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1)
assert_equal(
transfer_limits['activity1'][get_rse_id(
self.old_rse_1)]['max_transfers'], 1)
assert_equal(
transfer_limits['activity2'][get_rse_id(
self.old_rse_1)]['max_transfers'], 2)
# Distances
distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0]
assert_equal(distance['ranking'], 10)
distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0]
assert_equal(distance['ranking'], 4)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data2))
assert_equal(r2.status, 201)
r2 = TestApp(import_app.wsgifunc(*mw)).post(
'/',
headers=headers2,
expect_errors=True,
params=render_json(**self.data3))
assert_equal(r2.status, 201)
