def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None,
exclude_rses=None, include_rses=None, delay_seconds=0, all_rses=False):
"""
Starts up the reaper threads.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param threads_per_worker: Total number of threads created by each worker.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
"""
logging.info('main: starting processes')
if all_rses:
rses = list_rses()
elif not rses:
rses = [rse['id'] for rse in rse_core.list_rses()]
threads = []
for worker in range(total_workers):
kwargs = {'worker_number': worker,
'total_workers': total_workers,
'rses': rses,
'once': once,
'chunk_size': chunk_size,
'scheme': scheme}
threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (worker, total_workers)))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
def __init__(self, path=''):
try:
self.model = json.load(open(path, 'r'))
except IOError:
self.model = {
"STANDARD_LINK": {
"r2": 0.0,
"rmse": 0.0,
"rate": 23399639.38262837,
"datalen": 0.0,
"overhead": 12.025538382153206,
"diskrw": 12046990.897099394
}
}
# Create RSE to site name dictionary
self._rse2site = {}
self._rseid2site = {}
self._site2rses = {}
self._site2rseids = {}
for rse in list_rses():
if rse['deleted'] is False:
attribs = list_rse_attributes(rse['rse'])
try:
self._rse2site[rse['rse']] = attribs['site']
self._rseid2site[rse['id']] = attribs['site']
except KeyError:
continue
if attribs['site'] not in list(self._site2rses.keys()):
self._site2rses[attribs['site']] = []
self._site2rseids[attribs['site']] = []
self._site2rses[attribs['site']].append(rse['rse'])
self._site2rseids[attribs['site']].append(rse['id'])
def get_rses_to_hostname_mapping():
"""
Return a dictionaries mapping the RSEs to the hostname of the SE
:returns: Dictionary with RSE_id as key and (hostname, rse_info) as value
"""
result = REGION.get('rse_hostname_mapping')
if result is NO_VALUE:
result = {}
all_rses = list_rses()
for rse in all_rses:
try:
rse_protocol = get_rse_protocols(rse_id=rse['id'])
except RSENotFound:
logging.log(
logging.WARNING,
'RSE deleted while constructing rse-to-hostname mapping. Skipping %s',
rse['rse'])
continue
for prot in rse_protocol['protocols']:
if prot['domains']['wan']['delete'] == 1:
result[rse['id']] = (prot['hostname'], rse_protocol)
if rse['id'] not in result:
logging.log(logging.WARNING,
'No default delete protocol for %s', rse['rse'])
REGION.set('rse_hostname_mapping', result)
return result
return result
def test_all_rse(self):
""" RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """
all_rses = rse.list_rses()
assert_equal(
sorted(rse_expression_parser.parse_expression("*"),
key=lambda rse: rse['rse']),
sorted(all_rses, key=lambda rse: rse['rse']))
def test_export_rest(self):
""" EXPORT (REST): Test the export of data."""
mw = []
headers1 = {
'X-Rucio-Account': 'root',
'X-Rucio-Username': '******',
'X-Rucio-Password': '******'
}
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass',
headers=headers1,
expect_errors=True)
token = str(r1.header('X-Rucio-Auth-Token'))
headers2 = {
'X-Rucio-Type': 'user',
'X-Rucio-Account': 'root',
'X-Rucio-Auth-Token': str(token)
}
r2 = TestApp(export_app.wsgifunc(*mw)).get('/',
headers=headers2,
expect_errors=True)
rses = [export_rse(rse['rse']) for rse in list_rses()]
assert_equal(r2.status, 200)
assert_equal(
r2.body,
render_json(**{
'rses': rses,
'distances': export_distances()
}))
def get_rses_to_process(rses, include_rses, exclude_rses, vos):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
If None, we either use all VOs if run from "def",
:returns: A list of RSEs to process
"""
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
if vos:
logging.log(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
vos = ['def']
else:
if vos:
invalid = set(vos) - set([v['vo'] for v in list_vos()])
if invalid:
msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid]))
raise VONotFound(msg)
else:
vos = [v['vo'] for v in list_vos()]
logging.log(logging.INFO, 'Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
cache_key = 'rses_to_process'
if multi_vo:
cache_key += '@%s' % '-'.join(vo for vo in vos)
result = REGION.get(cache_key)
if result is not NO_VALUE:
return result
all_rses = []
for vo in vos:
all_rses.extend(list_rses(filters={'vo': vo}))
if rses:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set(cache_key, rses)
logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses]))
return rses
def test_all_rse(self):
""" RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """
all_rses = rse.list_rses(filters=self.filter['filter_'])
value = sorted(rse_expression_parser.parse_expression(
"*", **self.filter),
key=lambda rse: rse['rse'])
expected = sorted(all_rses, key=lambda rse: rse['rse'])
assert value == expected
def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None, vos=None, logger=logging.log):
"""
Get a list of rses for conveyor
:param rses: List of rses (Single-VO only)
:param include_rses: RSEs to include
:param exclude_rses: RSEs to exclude
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
If None, we either use all VOs if run from "def", or the current VO otherwise.
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:return: List of working rses
"""
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
if vos:
logger(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
vos = ['def']
else:
if vos:
invalid = set(vos) - set([v['vo'] for v in list_vos()])
if invalid:
msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid]))
raise VONotFound(msg)
else:
vos = [v['vo'] for v in list_vos()]
logger(logging.INFO, 'This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
working_rses = []
rses_list = []
for vo in vos:
rses_list.extend(list_rses(filters={'vo': vo}))
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
for vo in vos:
try:
parsed_rses = parse_expression(include_rses, filter={'vo': vo}, session=None)
except InvalidRSEExpression:
logger(logging.ERROR, "Invalid RSE exception %s to include RSEs", include_rses)
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression as error:
logger(logging.ERROR, "Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error)
else:
working_rses = [rse for rse in working_rses if rse not in parsed_rses]
working_rses = [rsemgr.get_rse_info(rse_id=rse['id']) for rse in working_rses]
return working_rses
def list_rses(filters=None):
"""
Lists all RSEs.
:param filters: dictionary of attributes by which the results should be filtered.
:returns: List of all RSEs.
"""
return rse_module.list_rses(filters=filters)
def test_export_client(self):
""" EXPORT (CLIENT): Test the export of data."""
export_client = ExportClient()
data = export_client.export_data()
rses = {}
for rse in list_rses():
rse_name = rse['rse']
rses[rse_name] = export_rse(rse_name)
assert_equal(data['rses'], parse_response(render_json(**rses)))
assert_equal(data['distances'], parse_response(render_json(**export_distances())))
def list_rses(filters=None):
"""
Lists all RSEs.
:param filters: dictionary of attributes by which the results should be filtered.
:returns: List of all RSEs.
"""
return rse_module.list_rses(filters=filters)
def list_rses(filters={}, vo='def'):
"""
Lists all RSEs.
:param filters: dictionary of attributes by which the results should be filtered.
:param vo: The VO to act on.
:returns: List of all RSEs.
"""
filters['vo'] = vo
return rse_module.list_rses(filters=filters)
def get_undeterministic_rses():
key = 'undeterministic_rses'
result = region.get(key)
if type(result) is NoValue:
rses_list = rse_core.list_rses(filters={'deterministic': 0})
result = [rse['id'] for rse in rses_list]
try:
region.set(key, result)
except:
logging.warning("Failed to set dogpile cache, error: %s" % (traceback.format_exc()))
return result
def resolve_elements(self, session):
"""
Inherited from :py:func:`BaseExpressionElement.resolve_elements`
"""
output = list_rses({self.key: self.value}, session=session)
if not output:
return (set(), {})
rse_dict = {}
for rse in output:
rse_dict[rse['id']] = rse
return (set([rse['id'] for rse in output]), rse_dict)
def resolve_elements(self, session):
"""
Inherited from :py:func:`BaseExpressionElement.resolve_elements`
"""
output = list_rses({self.key: self.value}, session=session)
if not output:
return (set(), {})
rse_dict = {}
for rse in output:
rse_dict[rse['id']] = rse
return (set([rse['id'] for rse in output]), rse_dict)
def export_rses(session=None):
"""
Export RSE data.
:param session: database session in use.
"""
data = {}
for rse in rse_module.list_rses(session=session):
rse_name = rse['rse']
data[rse_name] = rse_module.export_rse(rse_name, session=session)
return data
def export_rses(vo='def', session=None):
"""
Export RSE data.
:param vo: The VO to export.
:param session: database session in use.
"""
data = {}
for rse in rse_module.list_rses(filters={'vo': vo}, session=session):
rse_id = rse['id']
data[rse_id] = rse_module.export_rse(rse_id, session=session)
return data
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0):
"""
Starts up the reaper threads.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param threads_per_worker: Total number of threads created by each worker.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
"""
logging.info('main: starting processes')
rses_list = rse_core.list_rses()
if rses:
rses = [rse for rse in rses_list if rse['rse'] in rses]
else:
rses = rses_list
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses]))
threads = []
nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1.0
for worker in xrange(total_workers):
for child in xrange(threads_per_worker or 1):
kwargs = {'worker_number': worker,
'child_number': child + 1,
'total_children': threads_per_worker or 1,
'once': once,
'chunk_size': chunk_size,
'greedy': greedy,
'rses': rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker],
'delay_seconds': delay_seconds,
'scheme': scheme}
threads.append(threading.Thread(target=reaper, kwargs=kwargs))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
def export_data(session=None):
"""
Export data.
:param session: database session in use.
"""
data = {
'rses': [
rse_module.export_rse(rse['rse'], session=session)
for rse in rse_module.list_rses(session=session)
],
'distances':
distance_module.export_distances(session=session)
}
return data
def get_rses(rses=None, include_rses=None, exclude_rses=None):
working_rses = []
rses_list = rse_core.list_rses()
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s to include RSEs" % (include_rses))
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
def get_rses_to_process(rses, include_rses, exclude_rses):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:returns: A list of RSEs to process
"""
result = REGION.get('rses_to_process')
if result is not NO_VALUE:
return result
all_rses = list_rses()
if rses:
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
logging.warning(
'Ignoring argument rses, this is only available in a single-VO setup. Please try an RSE Expression with include_rses if it is required.'
)
rses = all_rses
else:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format(
's' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set('rses_to_process', rses)
logging.info('Reaper: This instance will work on RSEs: %s',
', '.join([rse['rse'] for rse in rses]))
return rses
def list_rses(filters={}, vo='def', session=None):
"""
Lists all RSEs.
:param filters: dictionary of attributes by which the results should be filtered.
:param vo: The VO to act on.
:param session: The database session in use.
:returns: List of all RSEs.
"""
if not filters:
filters = {}
filters['vo'] = vo
return rse_module.list_rses(filters=filters, session=session)
def __get_undeterministic_rses():
"""
Get the undeterministic rses from the database
:returns: List of undeterministc rses
"""
key = 'undeterministic_rses'
result = region.get(key)
if isinstance(result, NoValue):
rses_list = list_rses(filters={'deterministic': False})
result = [rse['id'] for rse in rses_list]
try:
region.set(key, result)
except Exception as error:
logging.warning("Failed to set dogpile cache, error: %s", str(error))
return result
def sync_accounts(self, iam_users):
for user in iam_users:
username = user['userName']
email = user['emails'][0]['value']
if not user['active']:
logging.debug(
'Skipped account creation for User {} [not active]'.format(
username))
continue
# Rucio DB schema restriction
if len(username) > 25:
logging.debug(
'Skipped account creation for User {} [len(username) > 25]'
.format(username))
continue
if not account.account_exists(InternalAccount(username)):
account.add_account(InternalAccount(username),
AccountType.SERVICE, email)
logging.debug(
'Created account for User {} ***'.format(username))
# Give account quota for all RSEs
for rse_obj in rse.list_rses():
set_local_account_limit(InternalAccount(username),
rse_obj['id'], 1000000000000)
# Make the user an admin & able to sign URLs
try:
add_account_attribute(InternalAccount(username), 'admin',
'True')
add_account_attribute(InternalAccount(username),
'sign-gcs', 'True')
except Exception as e:
logging.debug(e)
if "groups" in user:
for group in user['groups']:
group_name = group['display']
if not account.has_account_attribute(
InternalAccount(username), group_name):
add_account_attribute(InternalAccount(username),
group_name, 'True')
def __get_undeterministic_rses():
"""
Get the undeterministic rses from the database
:returns: List of undeterministc rses
"""
key = 'undeterministic_rses'
result = region.get(key)
if type(result) is NoValue:
rses_list = list_rses(filters={'deterministic': False})
result = [rse['id'] for rse in rses_list]
try:
region.set(key, result)
except:
logging.warning("Failed to set dogpile cache, error: %s" %
(traceback.format_exc()))
return result
def get_authfile_prefixes(self):
session = get_session()
session.connection()
prefixes = set()
for rse in list_rses():
rse_id = rse['id']
protocols = get_rse_protocols(rse_id)
if 'protocols' in protocols:
for protocol in protocols['protocols']:
scheme = protocol['scheme']
hostname = protocol['hostname']
port = protocol['port']
prefix = self.clean_prefix_path(protocol['prefix'])
authfile_prefix = f"/{scheme}:/{hostname}:{port}/{prefix}"
prefixes.add(authfile_prefix)
return list(prefixes)
def __get_unavailable_read_rse_ids(session=None):
"""
Get unavailable read rse ids
"""
key = 'unavailable_read_rse_ids'
result = REGION_SHORT.get(key)
if type(result) is NoValue:
try:
logging.debug("Refresh unavailable read rses")
unavailable_read_rses = list_rses(filters={'availability_read': False}, session=session)
unavailable_read_rse_ids = [r['id'] for r in unavailable_read_rses]
REGION_SHORT.set(key, unavailable_read_rse_ids)
return unavailable_read_rse_ids
except Exception:
logging.warning("Failed to refresh unavailable read rses, error: %s" % (traceback.format_exc()))
return []
return result
def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None):
"""
Get a list of rses for conveyor
:param rses: List of rses
:param include_rses: RSEs to include
:param exclude_rses: RSEs to exclude
:return: List of working rses
"""
working_rses = []
rses_list = list_rses()
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to include RSEs" %
(include_rses))
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to exclude RSEs: %s" %
(exclude_rses, error))
else:
working_rses = [
rse for rse in working_rses if rse not in parsed_rses
]
working_rses = [rsemgr.get_rse_info(rse['rse']) for rse in working_rses]
return working_rses
def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None):
"""
Get a list of rses for conveyor
:param rses: List of rses (Single-VO only)
:param include_rses: RSEs to include
:param exclude_rses: RSEs to exclude
:return: List of working rses
"""
working_rses = []
rses_list = list_rses()
if rses:
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
logging.warning('Ignoring argument rses, this is only available in a single-vo setup. Please try an RSE Expression with include_rses if it is required.')
else:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to include RSEs", include_rses)
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error)
else:
working_rses = [rse for rse in working_rses if rse not in parsed_rses]
working_rses = [rsemgr.get_rse_info(rse['id']) for rse in working_rses]
return working_rses
def __get_unavailable_rse_ids(operation, session=None):
"""
Get unavailable rse ids for a given operation : read, write, delete
"""
if operation not in ['read', 'write', 'delete']:
logging.error("Wrong operation specified : %s" % (operation))
return []
key = 'unavailable_%s_rse_ids' % operation
result = REGION_SHORT.get(key)
if isinstance(result, NoValue):
try:
logging.debug("Refresh unavailable %s rses" % operation)
availability_key = 'availability_%s' % operation
unavailable_rses = list_rses(filters={availability_key: False}, session=session)
unavailable_rse_ids = [rse['id'] for rse in unavailable_rses]
REGION_SHORT.set(key, unavailable_rse_ids)
return unavailable_rse_ids
except Exception:
logging.warning("Failed to refresh unavailable %s rses, error: %s" % (operation, traceback.format_exc()))
return []
return result
def get_rses_to_process(rses, include_rses, exclude_rses):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:returns: A list of RSEs to process
"""
result = REGION.get('rses_to_process')
if result is not NO_VALUE:
return result
all_rses = list_rses()
if rses:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format(
's' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set('rses_to_process', rses)
logging.info('Reaper: This instance will work on RSEs: %s',
', '.join([rse['rse'] for rse in rses]))
return rses
def get_sources(dest_rse, scheme, req):
allowed_rses = []
if req['request_type'] == RequestType.STAGEIN:
rses = rse_core.list_rses(filters={'staging_buffer': dest_rse['rse']})
allowed_rses = [x['rse'] for x in rses]
allowed_source_rses = []
if req['attributes']:
if type(req['attributes']) is dict:
req_attributes = json.loads(json.dumps(req['attributes']))
else:
req_attributes = json.loads(str(req['attributes']))
source_replica_expression = req_attributes["source_replica_expression"]
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s for request %s: %s" % (source_replica_expression,
req['request_id'],
e))
allowed_source_rses = []
else:
allowed_source_rses = [x['rse'] for x in parsed_rses]
def get_rses_to_hostname_mapping():
"""
Return a dictionaries mapping the RSEs to the hostname of the SE
:returns: Dictionary with RSE_id as key and (hostname, rse_info) as value
"""
result = REGION.get('rse_hostname_mapping')
if result is NO_VALUE:
result = {}
all_rses = list_rses()
for rse in all_rses:
rse_protocol = get_rse_protocols(rse_id=rse['id'])
for prot in rse_protocol['protocols']:
if prot['domains']['wan']['delete'] == 1:
result[rse['id']] = (prot['hostname'], rse_protocol)
if rse['id'] not in result:
logging.warn('No default delete protocol for %s', rse['rse'])
REGION.set('rse_hostname_mapping', result)
return result
return result
def run(one_worker_per_rse=False, once=False, rses=[], scheme=None, all_os_rses=False, older_than=30, sleep_time=1):
"""
Starts up the injector threads.
:param one_worker_per_rse: If True, one worker per RSE; Otherwise, one worker for all RSEs.
:param once: If True, only runs one iteration of the main loop.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param all_os_rses: All Objectstore RSEs.
:param older_than: List control: older objects more than this value of days to list.
:param sleep_time: Days to sleep.
"""
logging.info('main: starting processes')
if all_os_rses:
rses = []
for rse in rse_core.list_rses():
if rse['rse'].endswith('_ES'):
rses.append(rse['rse'])
threads = []
if one_worker_per_rse:
worker = 0
for rse in rses:
kwargs = {'once': once, 'rses': [rse], 'scheme': scheme, 'worker_number': worker, 'total_workers': len(rses),
'older_than': older_than, 'sleep_time': sleep_time}
threads.append(threading.Thread(target=injector, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (worker, len(rses))))
worker += 1
else:
kwargs = {'once': once, 'rses': rses, 'scheme': scheme, 'older_than': older_than, 'sleep_time': sleep_time}
threads.append(threading.Thread(target=injector, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (0, 1)))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
from dogpile.cache import make_region
from rucio.core.rse import list_rses, get_rse_protocols
rse_region = make_region().configure(
'dogpile.cache.memcached', # 'dogpile.cache.memory'
expiration_time=1,
arguments={'url': "127.0.0.1:11211", 'distributed_lock': True},
)
if __name__ == '__main__':
for rse in list_rses():
rse_info = get_rse_protocols(rse['rse'])
print rse['rse'], rse_info
rse_region.set(str(rse['rse']), rse_info)
# rse_region.delete(str(rse['rse']))
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, vos=None, delay_seconds=0):
"""
Starts up the reaper threads.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param threads_per_worker: Total number of threads created by each worker.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
If None, we either use all VOs if run from "def", or the current VO otherwise.
"""
if rucio.db.sqla.util.is_old_db():
raise DatabaseException('Database was not updated, daemon won\'t start')
logging.info('Reaper1 daemon will be deprecated and replaced by reaper2 with Rucio release 1.25 (~March 2021)!')
logging.info('main: starting processes')
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
if vos:
logging.warning('Ignoring argument vos, this is only applicable in a multi-VO setup.')
vos = ['def']
else:
if vos:
invalid = set(vos) - set([v['vo'] for v in list_vos()])
if invalid:
msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid]))
raise VONotFound(msg)
else:
vos = [v['vo'] for v in list_vos()]
logging.info('Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
all_rses = []
for vo in vos:
all_rses.extend(rse_core.list_rses(filters={'vo': vo}))
if rses:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if not rses:
logging.error('Reaper: No RSEs found. Exiting.')
return
logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses]))
threads = []
nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1
rses = random.sample(rses, len(rses))
for worker in range(total_workers):
for child in range(threads_per_worker or 1):
rses_list = rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker]
if not rses_list:
logging.warning('Reaper: Empty RSEs list for worker %(worker)s' % locals())
continue
kwargs = {'worker_number': worker,
'child_number': child,
'total_children': threads_per_worker or 1,
'once': once,
'chunk_size': chunk_size,
'greedy': greedy,
'rses': rses_list,
'delay_seconds': delay_seconds,
'scheme': scheme}
threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, child: %s' % (worker, child)))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
