def test_2(self):
# local quota not enough -> error
copies = 2
rses = [self.rse_1, self.rse_2]
set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=10)
increase(self.mock1_id, self.account, 10, 10)
update_account_counter(account=self.account, rse_id=self.mock1_id)
with assert_raises(InsufficientAccountLimit):
RSESelector(self.account, rses, None, copies)
def test_5(self):
# enough RSEs and local quota, but global quota missing -> 1 RSE
copies = 1
rses = [self.rse_1, self.rse_2]
set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=10)
increase(self.mock1_id, self.account, 10, 10)
update_account_counter(account=self.account, rse_id=self.mock1_id)
set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20)
set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20)
rse_selector = RSESelector(self.account, rses, None, copies)
assert_equal(len(rse_selector.rses), 1)
def account_update(once=False,
process=0,
total_processes=1,
thread=0,
threads_per_process=1):
"""
Main loop to check and update the Account Counters.
"""
logging.info('account_update: starting')
logging.info('account_update: started')
while not graceful_stop.is_set():
try:
# Select a bunch of rses for to update for this worker
start = time.time() # NOQA
account_rse_ids = get_updated_account_counters(
total_workers=total_processes * threads_per_process - 1,
worker_number=process * threads_per_process + thread)
logging.debug('Index query time %f size=%d' %
(time.time() - start, len(account_rse_ids)))
# If the list is empty, sent the worker to sleep
if not account_rse_ids and not once:
logging.info('account_update[%s/%s] did not get any work' %
(process * threads_per_process + thread,
total_processes * threads_per_process - 1))
time.sleep(10)
else:
for account_rse_id in account_rse_ids:
if graceful_stop.is_set():
break
start_time = time.time()
update_account_counter(account=account_rse_id[0],
rse_id=account_rse_id[1])
logging.debug(
'account_update[%s/%s]: update of account-rse counter "%s-%s" took %f'
% (process * threads_per_process + thread,
total_processes * threads_per_process - 1,
account_rse_id[0], account_rse_id[1],
time.time() - start_time))
except Exception:
logging.error(traceback.format_exc())
if once:
break
logging.info('account_update: graceful stop requested')
logging.info('account_update: graceful stop done')
def test_3(self):
# global quota not enough -> error
copies = 2
rses = [self.rse_1, self.rse_2]
set_local_account_limit(account=self.account,
rse_id=self.mock1_id,
bytes=20)
set_global_account_limit(account=self.account,
rse_expression=self.rse_1_name,
bytes=10)
increase(self.mock1_id, self.account, 10, 10)
update_account_counter(account=self.account, rse_id=self.mock1_id)
with pytest.raises(InsufficientAccountLimit):
RSESelector(self.account, rses, None, copies)
def account_update(once=False):
"""
Main loop to check and update the Account Counters.
"""
logging.info('account_update: starting')
logging.info('account_update: started')
# Make an initial heartbeat so that all abacus-account daemons have the correct worker number on the next try
executable = 'abacus-account'
hostname = socket.gethostname()
pid = os.getpid()
current_thread = threading.current_thread()
live(executable=executable, hostname=hostname, pid=pid, thread=current_thread)
while not graceful_stop.is_set():
try:
# Heartbeat
heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=current_thread)
# Select a bunch of rses for to update for this worker
start = time.time() # NOQA
account_rse_ids = get_updated_account_counters(total_workers=heartbeat['nr_threads'],
worker_number=heartbeat['assign_thread'])
logging.debug('Index query time %f size=%d' % (time.time() - start, len(account_rse_ids)))
# If the list is empty, sent the worker to sleep
if not account_rse_ids and not once:
logging.info('account_update[%s/%s] did not get any work' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1))
time.sleep(10)
else:
for account_rse_id in account_rse_ids:
if graceful_stop.is_set():
break
start_time = time.time()
update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1])
logging.debug('account_update[%s/%s]: update of account-rse counter "%s-%s" took %f' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1, account_rse_id[0], account_rse_id[1], time.time() - start_time))
except Exception:
logging.error(traceback.format_exc())
if once:
break
logging.info('account_update: graceful stop requested')
die(executable=executable, hostname=hostname, pid=pid, thread=current_thread)
logging.info('account_update: graceful stop done')
def run_once(heartbeat_handler, **_kwargs):
worker_number, total_workers, logger = heartbeat_handler.live()
start = time.time() # NOQA
account_rse_ids = get_updated_account_counters(total_workers=total_workers,
worker_number=worker_number)
logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(account_rse_ids)))
# If the list is empty, sent the worker to sleep
if not account_rse_ids:
logger(logging.INFO, 'did not get any work')
return
for account_rse_id in account_rse_ids:
worker_number, total_workers, logger = heartbeat_handler.live()
if graceful_stop.is_set():
break
start_time = time.time()
update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1])
logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_rse_id[0], account_rse_id[1], time.time() - start_time))
def account_update(once=False, process=0, total_processes=1, thread=0, threads_per_process=1):
"""
Main loop to check and update the Account Counters.
"""
logging.info('account_update: starting')
logging.info('account_update: started')
while not graceful_stop.is_set():
try:
# Select a bunch of rses for to update for this worker
start = time.time() # NOQA
account_rse_ids = get_updated_account_counters(total_workers=total_processes*threads_per_process-1,
worker_number=process*threads_per_process+thread)
logging.debug('Index query time %f size=%d' % (time.time() - start, len(account_rse_ids)))
# If the list is empty, sent the worker to sleep
if not account_rse_ids and not once:
logging.info('account_update[%s/%s] did not get any work' % (process*threads_per_process+thread, total_processes*threads_per_process-1))
time.sleep(10)
else:
for account_rse_id in account_rse_ids:
if graceful_stop.is_set():
break
start_time = time.time()
update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1])
logging.debug('account_update[%s/%s]: update of account-rse counter "%s-%s" took %f' % (process*threads_per_process+thread, total_processes*threads_per_process-1, account_rse_id[0], account_rse_id[1], time.time() - start_time))
except Exception:
logging.error(traceback.format_exc())
if once:
break
logging.info('account_update: graceful stop requested')
logging.info('account_update: graceful stop done')
def test_api_rse(self):
""" RSE (API): Test external representation of RSEs """
out = api_rse.get_rse(self.rse_name, **self.vo)
assert out['rse'] == self.rse_name
assert out['id'] == self.rse_id
out = api_rse.list_rses(**self.new_vo)
out = list(out)
assert 0 != len(out)
rse_ids = [rse['id'] for rse in out]
assert self.rse3_id in rse_ids
assert self.rse4_id in rse_ids
for rse in out:
assert 'rse' in rse
if rse['id'] == self.rse3_id:
assert rse['rse'] == self.rse3_name
elif rse['id'] == self.rse4_id:
assert rse['rse'] == self.rse4_name
key = "KEY_" + generate_uuid()
api_rse.add_rse_attribute(self.rse_name,
key,
1,
issuer='root',
**self.vo)
out = api_rse.get_rses_with_attribute(key)
out = list(out)
assert 0 != len(out)
for rse in out:
assert rse['rse'] == self.rse_name
out = api_rse.get_rse_protocols(self.rse_name,
issuer='root',
**self.vo)
assert out['rse'] == self.rse_name
# add some account and RSE counters
rse_mock = 'MOCK4'
rse_mock_id = get_rse_id(rse_mock, **self.vo)
account_counter.del_counter(rse_id=rse_mock_id, account=self.account)
account_counter.add_counter(rse_id=rse_mock_id, account=self.account)
account_counter.increase(rse_id=rse_mock_id,
account=self.account,
files=1,
bytes=10)
account_counter.update_account_counter(self.account, rse_mock_id)
did = 'file_' + generate_uuid()
add_did(self.scope_name,
did,
'DATASET',
'root',
account=self.account_name,
rse=rse_mock,
**self.vo)
abacus_rse.run(once=True)
out = api_rse.get_rse_usage(rse_mock,
per_account=True,
issuer='root',
**self.vo)
assert rse_mock_id in [o['rse_id'] for o in out]
for usage in out:
if usage['rse_id'] == rse_mock_id:
assert usage['rse'] == rse_mock
accounts = [u['account'] for u in usage['account_usages']]
assert self.account_name in accounts
if self.multi_vo:
assert self.account.internal not in accounts
# clean up files
cleaner.run(once=True)
if self.multi_vo:
reaper.run(once=True,
include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock),
greedy=True)
else:
reaper.run(once=True, include_rses=rse_mock, greedy=True)
abacus_rse.run(once=True)
out = api_rse.parse_rse_expression(
'%s|%s' % (self.rse_name, self.rse2_name), **self.vo)
assert self.rse_name in out
assert self.rse2_name in out
assert self.rse_id not in out
assert self.rse2_id not in out
