def get_global_account_limits(account=None, session=None):
"""
Returns the global account limits for the account.
:param account: Account to check the limit for.
:param session: Database session in use.
:return: Dict {'MOCK': {'resolved_rses': ['MOCK'], 'limit': 10, 'resolved_rse_ids': [123]}}.
"""
if account:
global_account_limits = session.query(
models.AccountGlobalLimit).filter_by(account=account).all()
else:
global_account_limits = session.query(models.AccountGlobalLimit).all()
resolved_global_account_limits = {}
for limit in global_account_limits:
if account:
resolved_rses = parse_expression(limit['rse_expression'],
filter_={'vo': account.vo},
session=session)
else:
resolved_rses = parse_expression(limit['rse_expression'],
session=session)
limit_in_bytes = limit['bytes']
if limit_in_bytes == -1:
limit_in_bytes = float('inf')
resolved_global_account_limits[limit['rse_expression']] = {
'resolved_rses':
[resolved_rse['rse'] for resolved_rse in resolved_rses],
'resolved_rse_ids':
[resolved_rse['id'] for resolved_rse in resolved_rses],
'limit':
limit_in_bytes
}
return resolved_global_account_limits
def test_list_on_availability(self):
""" RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter"""
rsewrite_name = rse_name_generator()
rsenowrite_name = rse_name_generator()
rsewrite_id = rse.add_rse(rsewrite_name, **self.vo)
rsenowrite_id = rse.add_rse(rsenowrite_name, **self.vo)
attribute = attribute_name_generator()
rse.add_rse_attribute(rsewrite_id, attribute, "de")
rse.add_rse_attribute(rsenowrite_id, attribute, "de")
rse.update_rse(rsewrite_id, {'availability_write': True})
rse.update_rse(rsenowrite_id, {'availability_write': False})
value = sorted([
item['id'] for item in rse_expression_parser.parse_expression(
"%s=de" % attribute, **self.filter)
])
expected = sorted([rsewrite_id, rsenowrite_id])
assert value == expected
filters = self.filter
filters['availability_write'] = True
value = sorted([
item['id'] for item in rse_expression_parser.parse_expression(
"%s=de" % attribute, filters)
])
expected = sorted([rsewrite_id])
assert value == expected
filters['availability_write'] = False
pytest.raises(RSEWriteBlocked, rse_expression_parser.parse_expression,
"%s=de" % attribute, filters)
def test_numeric_operators(self):
""" RSE_EXPRESSION_PARSER (CORE) Test RSE attributes with numeric operations """
value = [
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s<11" % self.attribute_numeric, **self.filter)
]
assert value == [self.rse1_id]
pytest.raises(InvalidRSEExpression,
rse_expression_parser.parse_expression,
"%s<9" % self.attribute_numeric, **self.filter)
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s<21" % self.attribute_numeric, **self.filter)
])
expected = sorted([self.rse1_id, self.rse2_id])
assert value == expected
value = [
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s>49" % self.attribute_numeric, **self.filter)
]
assert value == [self.rse5_id]
pytest.raises(InvalidRSEExpression,
rse_expression_parser.parse_expression,
"%s>51" % self.attribute_numeric, **self.filter)
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s>30" % self.attribute_numeric, **self.filter)
])
expected = sorted([self.rse4_id, self.rse5_id])
assert value == expected
def test_numeric_operators(self):
""" RSE_EXPRESSION_PARSER (CORE) Test RSE attributes with numeric operations """
assert_equal([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s<11" % self.attribute_numeric)
], [self.rse1_id])
assert_raises(InvalidRSEExpression,
rse_expression_parser.parse_expression,
"%s<9" % self.attribute_numeric)
assert_equal(
sorted([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression(
"%s<21" % self.attribute_numeric)
]), sorted([self.rse1_id, self.rse2_id]))
assert_equal([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s>49" % self.attribute_numeric)
], [self.rse5_id])
assert_raises(InvalidRSEExpression,
rse_expression_parser.parse_expression,
"%s>51" % self.attribute_numeric)
assert_equal(
sorted([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression(
"%s>30" % self.attribute_numeric)
]), sorted([self.rse4_id, self.rse5_id]))
def test_list_on_availability():
""" RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter"""
rsewrite_name = rse_name_generator()
rsenowrite_name = rse_name_generator()
rsewrite_id = rse.add_rse(rsewrite_name)
rsenowrite_id = rse.add_rse(rsenowrite_name)
attribute = attribute_name_generator()
rse.add_rse_attribute(rsewrite_id, attribute, "de")
rse.add_rse_attribute(rsenowrite_id, attribute, "de")
rse.update_rse(rsewrite_id, {'availability_write': True})
rse.update_rse(rsenowrite_id, {'availability_write': False})
assert_equal(
sorted([
item['id']
for item in rse_expression_parser.parse_expression("%s=de" %
attribute)
]), sorted([rsewrite_id, rsenowrite_id]))
assert_equal(
sorted([
item['id'] for item in rse_expression_parser.parse_expression(
"%s=de" % attribute, {'availability_write': True})
]), sorted([rsewrite_id]))
assert_raises(RSEBlacklisted, rse_expression_parser.parse_expression,
"%s=de" % attribute, {'availability_write': False})
def get_rses_to_process(rses, include_rses, exclude_rses, vos):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
If None, we either use all VOs if run from "def",
:returns: A list of RSEs to process
"""
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
if vos:
logging.log(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
vos = ['def']
else:
if vos:
invalid = set(vos) - set([v['vo'] for v in list_vos()])
if invalid:
msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid]))
raise VONotFound(msg)
else:
vos = [v['vo'] for v in list_vos()]
logging.log(logging.INFO, 'Reaper: This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
cache_key = 'rses_to_process'
if multi_vo:
cache_key += '@%s' % '-'.join(vo for vo in vos)
result = REGION.get(cache_key)
if result is not NO_VALUE:
return result
all_rses = []
for vo in vos:
all_rses.extend(list_rses(filters={'vo': vo}))
if rses:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set(cache_key, rses)
logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses]))
return rses
def test_order_of_operations(self):
""" RSE_EXPRESSION_PARSER (CORE) Test order of operations """
value = sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute), **self.filter)])
expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id])
assert value == expected
value = sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute), **self.filter)])
expected = sorted([self.rse1_id, self.rse2_id])
assert value == expected
def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None, vos=None, logger=logging.log):
"""
Get a list of rses for conveyor
:param rses: List of rses (Single-VO only)
:param include_rses: RSEs to include
:param exclude_rses: RSEs to exclude
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
If None, we either use all VOs if run from "def", or the current VO otherwise.
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:return: List of working rses
"""
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
if not multi_vo:
if vos:
logger(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
vos = ['def']
else:
if vos:
invalid = set(vos) - set([v['vo'] for v in list_vos()])
if invalid:
msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '', ', '.join([repr(v) for v in invalid]))
raise VONotFound(msg)
else:
vos = [v['vo'] for v in list_vos()]
logger(logging.INFO, 'This instance will work on VO%s: %s' % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
working_rses = []
rses_list = []
for vo in vos:
rses_list.extend(list_rses(filters={'vo': vo}))
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
for vo in vos:
try:
parsed_rses = parse_expression(include_rses, filter={'vo': vo}, session=None)
except InvalidRSEExpression:
logger(logging.ERROR, "Invalid RSE exception %s to include RSEs", include_rses)
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression as error:
logger(logging.ERROR, "Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error)
else:
working_rses = [rse for rse in working_rses if rse not in parsed_rses]
working_rses = [rsemgr.get_rse_info(rse_id=rse['id']) for rse in working_rses]
return working_rses
def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None):
"""
Select a new target RSE for a rebalanced rule.
:param parent_rule rule that is rebalanced.
:param current_rse_id: RSE of the source.
:param rse_expression: RSE Expression of the source rule.
:param subscription_id: Subscription ID of the source rule.
:param rse_attributes: The attributes of the source rse.
:param other_rses: Other RSEs with existing dataset replicas.
:param exclude_expression: Exclude this rse_expression from being target_rses.
:param force_expression: Force a specific rse_expression as target.
:param session: The DB Session
:returns: New RSE expression
"""
if rse_attributes['type'] != 'DATADISK' and force_expression is None:
print('WARNING: dest RSE(s) has to be provided with --force-expression for rebalancing of non-datadisk RSES.')
raise InsufficientTargetRSEs
current_rse = get_rse_name(rse_id=current_rse_id)
current_rse_expr = current_rse
# if parent rule has a vo, enforce it
vo = parent_rule['scope'].vo
if exclude_expression:
target_rse = '(%s)\\%s' % (exclude_expression, current_rse_expr)
else:
target_rse = current_rse_expr
rses = parse_expression(expression=rse_expression, filter={'vo': vo}, session=session)
# TODO: dest rse selection should be configurable, there might be cases when tier is not defined, or concept of DATADISKS is not present.
# if subscription_id:
# pass
# # get_subscription_by_id(subscription_id, session)
if force_expression is not None:
if parent_rule['grouping'] != RuleGrouping.NONE:
rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session)
else:
# in order to avoid replication of the part of distributed dataset not present at rabalanced rse -> rses in force_expression
# this will be extended with development of delayed rule
rses = parse_expression(expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter={'vo': vo, 'availability_write': True}, session=session)
elif len(rses) > 1:
# Just define the RSE Expression without the current_rse
return '(%s)\\%s' % (rse_expression, target_rse)
else:
if rse_attributes['tier'] is True or int(rse_attributes['tier']) == 1:
# Tier 1 should go to another Tier 1
expression = '(tier=1&type=DATADISK)\\{}'.format(target_rse)
elif int(rse_attributes['tier']) == 2:
# Tier 2 should go to another Tier 2
expression = '(tier=2&type=DATADISK)\\{}'.format(target_rse)
elif int(rse_attributes['tier']) == 3:
# Tier 3 will go to Tier 2, since we don't have enough t3s
expression = '((tier=2&type=DATADISK)\\datapolicynucleus=1)\\{}'.format(target_rse)
rses = parse_expression(expression=expression, filter={'vo': vo, 'availability_write': True}, session=session)
rseselector = RSESelector(account=InternalAccount('ddmadmin', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session)
return get_rse_name([rse_id for rse_id, _, _ in rseselector.select_rse(size=0, preferred_rse_ids=[], blocklist=other_rses)][0], session=session)
def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None,
exclude_rses=None, include_rses=None, delay_seconds=0, all_rses=False):
"""
Starts up the reaper threads.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param threads_per_worker: Total number of threads created by each worker.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs (Single-VO only).
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
"""
logging.info('main: starting processes')
all_rses = list_rses()
if all_rses:
rses = all_rses
else:
if rses:
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
logging.warning('Ignoring argument rses, this is only available in a single-vo setup. Please try an RSE Expression with include_rses if it is required.')
rses = []
else:
rses = [rse_core.get_rse_id(rse=rse) for rse in rses]
rses = [rse for rse in rses if rse in all_rses]
else:
rses = all_rses
if exclude_rses:
excluded_rses = [rse['id'] for rse in parse_expression(exclude_rses)]
rses = [rse for rse in rses if rse not in excluded_rses]
if include_rses:
included_rses = [rse['id'] for rse in parse_expression(include_rses)]
rses = [rse for rse in rses if rse in included_rses]
if not rses:
logging.error('Dark Reaper: No RSEs found. Exiting.')
return
threads = []
for worker in range(total_workers):
kwargs = {'worker_number': worker,
'total_workers': total_workers,
'rses': rses,
'once': once,
'chunk_size': chunk_size,
'scheme': scheme}
threads.append(threading.Thread(target=reaper, kwargs=kwargs, name='Worker: %s, Total_Workers: %s' % (worker, total_workers)))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
def test_rses_at_different_vos(self):
""" MULTI VO (CLIENT): Test that RSEs from 2nd vo don't interfere """
# Set up RSEs at two VOs
rse_client = RSEClient()
rse_str = ''.join(choice(ascii_uppercase) for x in range(10))
tst = 'TST_%s' % rse_str
new = 'NEW_%s' % rse_str
shr = 'SHR_%s' % rse_str
rse_client.add_rse(tst)
rse_client.add_rse(shr)
add_rse(new, 'root', **self.new_vo)
shr_id_new_original = add_rse(shr, 'root', **self.new_vo) # Accurate rse_id for shared RSE at 'new'
# Check the cached rse-id from each VO does not interfere
shr_id_tst = get_rse_id(shr, **self.vo)
shr_id_new = get_rse_id(shr, **self.new_vo)
assert_equal(shr_id_new, shr_id_new_original)
assert_not_equal(shr_id_new, shr_id_tst)
# Check that when listing RSEs we only get RSEs for our VO
rse_list_tst = [r['rse'] for r in rse_client.list_rses()]
rse_list_new = [r['rse'] for r in list_rses(filters={}, **self.new_vo)]
assert_true(tst in rse_list_tst)
assert_false(new in rse_list_tst)
assert_true(shr in rse_list_tst)
assert_false(tst in rse_list_new)
assert_true(new in rse_list_new)
assert_true(shr in rse_list_new)
# Check the cached attribute-value results do not interfere and only give results from the appropriate VO
attribute_value = generate_uuid()
add_rse_attribute(new, 'test', attribute_value, 'root', **self.new_vo)
rses_tst_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_1 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
rses_tst_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.vo))
rses_new_2 = list(get_rses_with_attribute_value('test', attribute_value, 'test', **self.new_vo))
assert_equal(len(rses_tst_1), 0)
assert_not_equal(len(rses_new_1), 0)
assert_equal(len(rses_tst_2), 0)
assert_not_equal(len(rses_new_2), 0)
# check parse_expression
rses_tst_3 = parse_expression(shr, filter={'vo': self.vo['vo']})
rses_tst_4 = parse_expression(tst, filter={'vo': self.vo['vo']})
rses_new_3 = parse_expression(shr, filter={'vo': self.new_vo['vo']})
with assert_raises(InvalidRSEExpression):
parse_expression(tst, filter={'vo': self.new_vo['vo']})
assert_equal(len(rses_tst_3), 1)
assert_equal(shr_id_tst, rses_tst_3[0]['id'])
assert_equal(len(rses_tst_4), 1)
assert_equal(tst, rses_tst_4[0]['rse'])
assert_equal(len(rses_new_3), 1)
assert_equal(shr_id_new, rses_new_3[0]['id'])
def test_order_of_operations(self):
""" RSE_EXPRESSION_PARSER (CORE) Test order of operations """
assert_equal(
sorted([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression(
"%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute))
]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
assert_equal(
sorted([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression(
"%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute))
]), sorted([self.rse1_id, self.rse2_id]))
def run(total_workers=1, chunk_size=100, threads_per_worker=None, once=False, greedy=False, rses=[], scheme=None, exclude_rses=None, include_rses=None, delay_seconds=0):
"""
Starts up the reaper threads.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param threads_per_worker: Total number of threads created by each worker.
:param once: If True, only runs one iteration of the main loop.
:param greedy: If True, delete right away replicas with tombstone.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
"""
logging.info('main: starting processes')
rses_list = rse_core.list_rses()
if rses:
rses = [rse for rse in rses_list if rse['rse'] in rses]
else:
rses = rses_list
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses]))
threads = []
nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1.0
for worker in xrange(total_workers):
for child in xrange(threads_per_worker or 1):
kwargs = {'worker_number': worker,
'child_number': child + 1,
'total_children': threads_per_worker or 1,
'once': once,
'chunk_size': chunk_size,
'greedy': greedy,
'rses': rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker],
'delay_seconds': delay_seconds,
'scheme': scheme}
threads.append(threading.Thread(target=reaper, kwargs=kwargs))
[t.start() for t in threads]
while threads[0].is_alive():
[t.join(timeout=3.14) for t in threads]
def get_rses_to_process(rses, include_rses, exclude_rses):
"""
Return the list of RSEs to process based on rses, include_rses and exclude_rses
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param exclude_rses: RSE expression to exclude RSEs from the Reaper.
:param include_rses: RSE expression to include RSEs.
:returns: A list of RSEs to process
"""
result = REGION.get('rses_to_process')
if result is not NO_VALUE:
return result
all_rses = list_rses()
if rses:
if config_get_bool('common',
'multi_vo',
raise_exception=False,
default=False):
logging.warning(
'Ignoring argument rses, this is only available in a single-VO setup. Please try an RSE Expression with include_rses if it is required.'
)
rses = all_rses
else:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format(
's' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
REGION.set('rses_to_process', rses)
logging.info('Reaper: This instance will work on RSEs: %s',
', '.join([rse['rse'] for rse in rses]))
return rses
def test_tag_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """
assert_equal(
sorted([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression(self.tag1)
]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_all_rse(self):
""" RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """
all_rses = rse.list_rses()
assert_equal(
sorted(rse_expression_parser.parse_expression("*"),
key=lambda rse: rse['rse']),
sorted(all_rses, key=lambda rse: rse['rse']))
def test_attribute_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """
assert_equal([
t_rse['id']
for t_rse in rse_expression_parser.parse_expression("%s=uk" %
self.attribute)
], [self.rse4_id])
def perm_add_rule(issuer, kwargs, session=None):
"""
Checks if an account can add a replication rule.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:param session: The DB session to use
:returns: True if account is allowed, otherwise False
"""
rses = parse_expression(kwargs['rse_expression'], filter_={'vo': issuer.vo}, session=session)
# Keep while sync is running so it can make rules on all RSEs
if _is_root(issuer) and repr(kwargs['account']).startswith('sync_'):
return True
if isinstance(repr(issuer), basestring) and repr(issuer).startswith('sync_'):
return True
# Anyone can use _Temp RSEs if a lifetime is set and under a month
all_temp = True
for rse in rses:
rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
rse_type = rse_attr.get('cms_type', None)
if rse_type not in ['temp']:
all_temp = False
if all_temp and kwargs['lifetime'] is not None and kwargs['lifetime'] < 31 * 24 * 60 * 60:
return True
if kwargs['account'] == issuer and not kwargs['locked']:
return True
if _is_root(issuer) or has_account_attribute(account=issuer, key='admin', session=session):
return True
return False
def __init__(self, datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas):
self._fsc = FreeSpaceCollector()
self._nmc = NetworkMetricsCollector()
self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
self._added_bytes = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_bytes_")
self._added_files = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_files_")
self._datatypes = datatypes.split(',')
self._dest_rse_expr = dest_rse_expr
self._max_bytes_hour = max_bytes_hour
self._max_files_hour = max_files_hour
self._max_bytes_hour_rse = max_bytes_hour_rse
self._max_files_hour_rse = max_files_hour_rse
self._min_popularity = min_popularity
self._min_recent_requests = min_recent_requests
self._max_replicas = max_replicas
rses = parse_expression(self._dest_rse_expr)
self._rses = {}
self._sites = {}
for rse in rses:
rse_attrs = list_rse_attributes(rse['rse'])
rse_attrs['rse'] = rse['rse']
self._rses[rse['rse']] = rse_attrs
self._sites[rse_attrs['site']] = rse_attrs
self._dst_penalties = {}
self._src_penalties = {}
self._print_params()
def perm_approve_rule(issuer, kwargs, session=None):
"""
Checks if an issuer can approve a replication rule.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:param session: The DB session to use
:returns: True if account is allowed to call the API call, otherwise False
"""
if _is_root(issuer) or has_account_attribute(
account=issuer, key='admin', session=session):
return True
rule = get_rule(rule_id=kwargs['rule_id'])
rses = parse_expression(rule['rse_expression'],
filter_={'vo': issuer.vo},
session=session)
# Those in rule_approvers can approve the rule
for rse in rses:
rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
rule_approvers = rse_attr.get('rule_approvers', None)
if rule_approvers and issuer.external in rule_approvers.split(','):
return True
return False
def perm_del_rule(issuer, kwargs):
"""
Checks if an issuer can delete a replication rule.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:returns: True if account is allowed to call the API call, otherwise False
"""
if issuer == 'root' or issuer == 'ddmadmin':
return True
if get_rule(kwargs['rule_id'])['account'] == issuer:
return True
# Check if user is a country admin
admin_in_country = []
for kv in list_account_attributes(account=issuer):
if kv['key'].startswith('country-') and kv['value'] == 'admin':
admin_in_country.append(kv['key'].partition('-')[2])
rule = get_rule(rule_id=kwargs['rule_id'])
rses = parse_expression(rule['rse_expression'])
if admin_in_country:
for rse in rses:
if list_rse_attributes(rse=None, rse_id=rse['id']).get('country') in admin_in_country:
return True
# DELETERS can approve the rule
for rse in rses:
rse_attr = list_rse_attributes(rse=None, rse_id=rse['id'])
if rse_attr.get('rule_deleters'):
if issuer in rse_attr.get('rule_deleters').split(','):
return True
return False
def perm_get_global_account_usage(issuer, kwargs):
"""
Checks if an account can get the account usage of an account.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:returns: True if account is allowed, otherwise False
"""
if _is_root(issuer) or has_account_attribute(
account=issuer, key='admin') or kwargs.get('account') == issuer:
return True
# Check if user is a country admin for all involved countries
admin_in_country = set()
for kv in list_account_attributes(account=issuer):
if kv['key'].startswith('country-') and kv['value'] == 'admin':
admin_in_country.add(kv['key'].partition('-')[2])
resolved_rse_countries = {
list_rse_attributes(rse_id=rse['rse_id']).get('country')
for rse in parse_expression(kwargs['rse_exp'],
filter={'vo': issuer.vo})
}
if resolved_rse_countries.issubset(admin_in_country):
return True
return False
def perm_add_rule(issuer, kwargs):
"""
Checks if an account can add a replication rule.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:returns: True if account is allowed, otherwise False
"""
rses = parse_expression(kwargs['rse_expression'])
# If all the RSEs matching the expression need approval, the rule cannot be created
if not kwargs['ask_approval']:
all_rses_need_approval = True
for rse in rses:
rse_attr = list_rse_attributes(rse_id=rse['id'])
if rse_attr.get('requires_approval', False):
all_rses_need_approval = False
if not all_rses_need_approval:
return False
if kwargs['account'] == issuer and not kwargs['locked']:
return True
if _is_root(issuer) or has_account_attribute(account=issuer, key='admin'):
return True
return False
def test_intersect(self):
""" RSE_EXPRESSION_PARSER (CORE) Test intersect operator """
value = [
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s&%s=uk" % (self.tag2, self.attribute), **self.filter)
]
assert value == [self.rse4_id]
def perm_delete_global_account_limit(issuer, kwargs, session=None):
"""
Checks if an account can delete a global account limit.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:param session: The DB session to use
:returns: True if account is allowed, otherwise False
"""
if _is_root(issuer) or has_account_attribute(
account=issuer, key='admin', session=session):
return True
# Check if user is a country admin
admin_in_country = set()
for kv in list_account_attributes(account=issuer, session=session):
if kv['key'].startswith('country-') and kv['value'] == 'admin':
admin_in_country.add(kv['key'].partition('-')[2])
if admin_in_country:
resolved_rse_countries = {
list_rse_attributes(rse_id=rse['rse_id'],
session=session).get('country')
for rse in parse_expression(kwargs['rse_expression'],
filter_={'vo': issuer.vo},
session=session)
}
if resolved_rse_countries.issubset(admin_in_country):
return True
return False
def test_simple_rse_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE reference """
value = [
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
self.rse1, **self.filter)
]
assert value == [self.rse1_id]
def test_attribute_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """
value = [
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s=uk" % self.attribute, **self.filter)
]
assert value == [self.rse4_id]
def test_tag_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
self.tag1, **self.filter)
])
expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id])
assert value == expected
def test_parantheses(self):
""" RSE_EXPRESSION_PARSER (CORE) Test parantheses """
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"(%s)" % self.tag1, **self.filter)
])
expected = sorted([self.rse1_id, self.rse2_id, self.rse3_id])
assert value == expected
def test_complement(self):
""" RSE_EXPRESSION_PARSER (CORE) Test complement operator """
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"%s\\%s" % (self.tag1, self.rse3), **self.filter)
])
expected = sorted([self.rse1_id, self.rse2_id])
assert value == expected
def test_complicated_expression_3(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 3"""
value = sorted([
t_rse['id'] for t_rse in rse_expression_parser.parse_expression(
"(*)&%s=at" % self.attribute, **self.filter)
])
expected = sorted([self.rse1_id])
assert value == expected
def parse_rse_expression(rse_expression):
"""
Parse an RSE expression and return the list of RSEs.
:param rse_expression: The RSE expression.
:returns: List of RSEs
:raises: InvalidRSEExpression
"""
rses = parse_expression(rse_expression)
return [rse['rse'] for rse in rses]
def test_list_rses_based_on_availability(self):
""" RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter"""
rseWRITE_name = rse_name_generator()
rseNOWRITE_name = rse_name_generator()
rseWRITE_id = rse.add_rse(rseWRITE_name)
rseNOWRITE_id = rse.add_rse(rseNOWRITE_name)
attribute = attribute_name_generator()
rse.add_rse_attribute(rseWRITE_name, attribute, "de")
rse.add_rse_attribute(rseNOWRITE_name, attribute, "de")
rse.update_rse(rseWRITE_name, {'availability_write': True})
rse.update_rse(rseNOWRITE_name, {'availability_write': False})
assert_equal(sorted([item['id'] for item in rse_expression_parser.parse_expression("%s=de" % attribute)]),
sorted([rseWRITE_id, rseNOWRITE_id]))
assert_equal(sorted([item['id'] for item in rse_expression_parser.parse_expression("%s=de" % attribute, {'availability_write': True})]),
sorted([rseWRITE_id]))
assert_raises(InvalidRSEExpression, rse_expression_parser.parse_expression, "%s=de" % attribute, {'availability_write': False})
def get_rses(rses=None, include_rses=None, exclude_rses=None):
working_rses = []
rses_list = rse_core.list_rses()
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s to include RSEs" % (include_rses))
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
def site_selector(replicas, site):
"""
Return a list of replicas located on one site.
:param replicas : A dict with RSEs as values and replicas as keys (URIs).
:param site : The name of the site
"""
result = []
try:
rses = parse_expression("site=%s" % site)
except InvalidRSEExpression:
return result
except Exception:
return result
rses = [i['rse'] for i in rses]
for replica in replicas:
if replicas[replica] in rses:
result.append(replica)
return result
def get_sources(dest_rse, scheme, req):
allowed_rses = []
if req['request_type'] == RequestType.STAGEIN:
rses = rse_core.list_rses(filters={'staging_buffer': dest_rse['rse']})
allowed_rses = [x['rse'] for x in rses]
allowed_source_rses = []
if req['attributes']:
if type(req['attributes']) is dict:
req_attributes = json.loads(json.dumps(req['attributes']))
else:
req_attributes = json.loads(str(req['attributes']))
source_replica_expression = req_attributes["source_replica_expression"]
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s for request %s: %s" % (source_replica_expression,
req['request_id'],
e))
allowed_source_rses = []
else:
allowed_source_rses = [x['rse'] for x in parsed_rses]
def test_tag_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression(self.tag1)]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_complement(self):
""" RSE_EXPRESSION_PARSER (CORE) Test complement operator """
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\%s" % (self.tag1, self.rse3))]), sorted([self.rse1_id, self.rse2_id]))
def test_intersect(self):
""" RSE_EXPRESSION_PARSER (CORE) Test intersect operator """
assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression("%s&%s=uk" % (self.tag2, self.attribute))], [self.rse4_id])
def test_complicated_expression_1(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 1"""
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(%s|%s)\\%s|%s&%s" % (self.tag1, self.tag2, self.tag2, self.tag2, self.tag1))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_complicated_expression_2(self):
""" RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 2"""
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" % (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]), sorted([self.rse1_id, self.rse2_id, self.rse5_id]))
def test_invalid_expression_unconnected_operator(self):
""" RSE_EXPRESSION_PARSER (CORE) Test invalid rse expression: unconnected operator"""
rse_expression_parser.parse_expression("TEST_RSE1|")
def test_invalid_expression_wrong_parantheses(self):
""" RSE_EXPRESSION_PARSER (CORE) Test invalid rse expression: wrong parantheses """
rse_expression_parser.parse_expression("TEST_RSE1)")
def test_unknown_RSE(self):
""" RSE_EXPRESSION_PARSER (CORE) Test unknown RSE """
rse_expression_parser.parse_expression("TEST_RSE999")
def test_simple_rse_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE reference """
assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression(self.rse1)], [self.rse1_id])
def test_attribute_reference(self):
""" RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """
assert_equal([rse['id'] for rse in rse_expression_parser.parse_expression("%s=uk" % self.attribute)], [self.rse4_id])
def check_rule(rule, session):
"""
Checks a rule
Returns a report object
"""
report = {'rule_id': rule.id,
'num_files': 0,
'did_type': None,
'num_datasets': None, # In case of a container
'missing_locks': 0,
'locks_placed_wrong': 0,
'missing_replicas': 0,
'failed_parse': 0}
try:
rse_set = parse_expression(expression=rule.rse_expression, session=session)
except:
report['failed_parse'] = 1
return report
did_type = session.query(models.DataIdentifier.did_type).filter_by(scope=rule.scope, name=rule.name).one()[0]
report['did_type'] = str(did_type)
if did_type == DIDType.CONTAINER:
report['num_datasets'] = 0
child_dids = list_child_dids(scope=rule.scope, name=rule.name, session=session)
for did in child_dids:
if did['type'] == DIDType.DATASET:
report['num_datasets'] += 1
files = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).filter_by(scope=did['scope'], name=did['name']).all()
report['num_files'] += len(files)
report = check_locks(report=report,
rule_id=rule.id,
copies=rule.copies,
rse_set=rse_set,
grouping=rule.grouping,
did=(did['scope'], did['name']),
did_type=DIDType.DATASET,
files=files,
session=session)
report = check_replicas(report=report,
copies=rule.copies,
rse_set=rse_set,
did=(did['scope'], did['name']),
did_type=DIDType.DATASET,
files=files,
session=session)
elif did_type == DIDType.DATASET:
files = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).filter_by(scope=rule.scope, name=rule.name).all()
report['num_files'] = len(files)
report = check_locks(report=report,
rule_id=rule.id,
copies=rule.copies,
rse_set=rse_set,
grouping=rule.grouping,
did=(rule.scope, rule.name),
did_type=DIDType.DATASET,
files=files,
session=session)
report = check_replicas(report=report,
copies=rule.copies,
rse_set=rse_set,
did=(rule.scope, rule.name),
did_type=DIDType.DATASET,
files=files,
session=session)
else:
report['num_files'] = 1
report = check_locks(report=report,
rule_id=rule.id,
copies=rule.copies,
rse_set=rse_set,
grouping=rule.grouping,
did=(rule.scope, rule.name),
did_type=DIDType.FILE,
files=[(rule.scope, rule.name)],
session=session)
report = check_replicas(report=report,
copies=rule.copies,
rse_set=rse_set,
did=(rule.scope, rule.name),
did_type=DIDType.DATASET,
files=[(rule.scope, rule.name)],
session=session)
return report
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s to include RSEs" % (include_rses))
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression, e:
logging.error("Invalid RSE exception %s to exclude RSEs: %s" % (exclude_rses, e))
else:
working_rses = [rse for rse in working_rses if rse not in parsed_rses]
working_rses = [rsemgr.get_rse_info(rse['rse']) for rse in working_rses]
return working_rses
def get_requests(rse_id=None, process=0, total_processes=1, thread=0, total_threads=1, mock=False, bulk=100, activity=None):
ts = time.time()
reqs = request.get_next(request_type=[RequestType.TRANSFER,
RequestType.STAGEIN,
RequestType.STAGEOUT],
state=RequestState.QUEUED,
def test_parantheses(self):
""" RSE_EXPRESSION_PARSER (CORE) Test parantheses """
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("(%s)" % self.tag1)]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
def test_order_of_operations(self):
""" RSE_EXPRESSION_PARSER (CORE) Test order of operations """
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\%s|%s=fr" % (self.tag1, self.rse3, self.attribute))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s\\(%s|%s=fr)" % (self.tag1, self.rse3, self.attribute))]), sorted([self.rse1_id, self.rse2_id]))
def test_union(self):
""" RSE_EXPRESSION_PARSER (CORE) Test union operator """
assert_equal(sorted([rse['id'] for rse in rse_expression_parser.parse_expression("%s|%s" % (self.tag1, self.tag2))]), sorted([self.rse1_id, self.rse2_id, self.rse3_id, self.rse4_id, self.rse5_id]))
