def test_2(self): # local quota not enough -> error copies = 2 rses = [self.rse_1, self.rse_2] set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=10) increase(self.mock1_id, self.account, 10, 10) update_account_counter(account=self.account, rse_id=self.mock1_id) with assert_raises(InsufficientAccountLimit): RSESelector(self.account, rses, None, copies)
def test_5(self): # enough RSEs and local quota, but global quota missing -> 1 RSE copies = 1 rses = [self.rse_1, self.rse_2] set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=10) increase(self.mock1_id, self.account, 10, 10) update_account_counter(account=self.account, rse_id=self.mock1_id) set_local_account_limit(account=self.account, rse_id=self.mock2_id, bytes=20) set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20) rse_selector = RSESelector(self.account, rses, None, copies) assert_equal(len(rse_selector.rses), 1)
def account_update(once=False, process=0, total_processes=1, thread=0, threads_per_process=1): """ Main loop to check and update the Account Counters. """ logging.info('account_update: starting') logging.info('account_update: started') while not graceful_stop.is_set(): try: # Select a bunch of rses for to update for this worker start = time.time() # NOQA account_rse_ids = get_updated_account_counters( total_workers=total_processes * threads_per_process - 1, worker_number=process * threads_per_process + thread) logging.debug('Index query time %f size=%d' % (time.time() - start, len(account_rse_ids))) # If the list is empty, sent the worker to sleep if not account_rse_ids and not once: logging.info('account_update[%s/%s] did not get any work' % (process * threads_per_process + thread, total_processes * threads_per_process - 1)) time.sleep(10) else: for account_rse_id in account_rse_ids: if graceful_stop.is_set(): break start_time = time.time() update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1]) logging.debug( 'account_update[%s/%s]: update of account-rse counter "%s-%s" took %f' % (process * threads_per_process + thread, total_processes * threads_per_process - 1, account_rse_id[0], account_rse_id[1], time.time() - start_time)) except Exception: logging.error(traceback.format_exc()) if once: break logging.info('account_update: graceful stop requested') logging.info('account_update: graceful stop done')
def test_3(self): # global quota not enough -> error copies = 2 rses = [self.rse_1, self.rse_2] set_local_account_limit(account=self.account, rse_id=self.mock1_id, bytes=20) set_global_account_limit(account=self.account, rse_expression=self.rse_1_name, bytes=10) increase(self.mock1_id, self.account, 10, 10) update_account_counter(account=self.account, rse_id=self.mock1_id) with pytest.raises(InsufficientAccountLimit): RSESelector(self.account, rses, None, copies)
def account_update(once=False): """ Main loop to check and update the Account Counters. """ logging.info('account_update: starting') logging.info('account_update: started') # Make an initial heartbeat so that all abacus-account daemons have the correct worker number on the next try executable = 'abacus-account' hostname = socket.gethostname() pid = os.getpid() current_thread = threading.current_thread() live(executable=executable, hostname=hostname, pid=pid, thread=current_thread) while not graceful_stop.is_set(): try: # Heartbeat heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=current_thread) # Select a bunch of rses for to update for this worker start = time.time() # NOQA account_rse_ids = get_updated_account_counters(total_workers=heartbeat['nr_threads'], worker_number=heartbeat['assign_thread']) logging.debug('Index query time %f size=%d' % (time.time() - start, len(account_rse_ids))) # If the list is empty, sent the worker to sleep if not account_rse_ids and not once: logging.info('account_update[%s/%s] did not get any work' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1)) time.sleep(10) else: for account_rse_id in account_rse_ids: if graceful_stop.is_set(): break start_time = time.time() update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1]) logging.debug('account_update[%s/%s]: update of account-rse counter "%s-%s" took %f' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1, account_rse_id[0], account_rse_id[1], time.time() - start_time)) except Exception: logging.error(traceback.format_exc()) if once: break logging.info('account_update: graceful stop requested') die(executable=executable, hostname=hostname, pid=pid, thread=current_thread) logging.info('account_update: graceful stop done')
def run_once(heartbeat_handler, **_kwargs): worker_number, total_workers, logger = heartbeat_handler.live() start = time.time() # NOQA account_rse_ids = get_updated_account_counters(total_workers=total_workers, worker_number=worker_number) logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(account_rse_ids))) # If the list is empty, sent the worker to sleep if not account_rse_ids: logger(logging.INFO, 'did not get any work') return for account_rse_id in account_rse_ids: worker_number, total_workers, logger = heartbeat_handler.live() if graceful_stop.is_set(): break start_time = time.time() update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1]) logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_rse_id[0], account_rse_id[1], time.time() - start_time))
def account_update(once=False, process=0, total_processes=1, thread=0, threads_per_process=1): """ Main loop to check and update the Account Counters. """ logging.info('account_update: starting') logging.info('account_update: started') while not graceful_stop.is_set(): try: # Select a bunch of rses for to update for this worker start = time.time() # NOQA account_rse_ids = get_updated_account_counters(total_workers=total_processes*threads_per_process-1, worker_number=process*threads_per_process+thread) logging.debug('Index query time %f size=%d' % (time.time() - start, len(account_rse_ids))) # If the list is empty, sent the worker to sleep if not account_rse_ids and not once: logging.info('account_update[%s/%s] did not get any work' % (process*threads_per_process+thread, total_processes*threads_per_process-1)) time.sleep(10) else: for account_rse_id in account_rse_ids: if graceful_stop.is_set(): break start_time = time.time() update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1]) logging.debug('account_update[%s/%s]: update of account-rse counter "%s-%s" took %f' % (process*threads_per_process+thread, total_processes*threads_per_process-1, account_rse_id[0], account_rse_id[1], time.time() - start_time)) except Exception: logging.error(traceback.format_exc()) if once: break logging.info('account_update: graceful stop requested') logging.info('account_update: graceful stop done')
def test_api_rse(self): """ RSE (API): Test external representation of RSEs """ out = api_rse.get_rse(self.rse_name, **self.vo) assert out['rse'] == self.rse_name assert out['id'] == self.rse_id out = api_rse.list_rses(**self.new_vo) out = list(out) assert 0 != len(out) rse_ids = [rse['id'] for rse in out] assert self.rse3_id in rse_ids assert self.rse4_id in rse_ids for rse in out: assert 'rse' in rse if rse['id'] == self.rse3_id: assert rse['rse'] == self.rse3_name elif rse['id'] == self.rse4_id: assert rse['rse'] == self.rse4_name key = "KEY_" + generate_uuid() api_rse.add_rse_attribute(self.rse_name, key, 1, issuer='root', **self.vo) out = api_rse.get_rses_with_attribute(key) out = list(out) assert 0 != len(out) for rse in out: assert rse['rse'] == self.rse_name out = api_rse.get_rse_protocols(self.rse_name, issuer='root', **self.vo) assert out['rse'] == self.rse_name # add some account and RSE counters rse_mock = 'MOCK4' rse_mock_id = get_rse_id(rse_mock, **self.vo) account_counter.del_counter(rse_id=rse_mock_id, account=self.account) account_counter.add_counter(rse_id=rse_mock_id, account=self.account) account_counter.increase(rse_id=rse_mock_id, account=self.account, files=1, bytes=10) account_counter.update_account_counter(self.account, rse_mock_id) did = 'file_' + generate_uuid() add_did(self.scope_name, did, 'DATASET', 'root', account=self.account_name, rse=rse_mock, **self.vo) abacus_rse.run(once=True) out = api_rse.get_rse_usage(rse_mock, per_account=True, issuer='root', **self.vo) assert rse_mock_id in [o['rse_id'] for o in out] for usage in out: if usage['rse_id'] == rse_mock_id: assert usage['rse'] == rse_mock accounts = [u['account'] for u in usage['account_usages']] assert self.account_name in accounts if self.multi_vo: assert self.account.internal not in accounts # clean up files cleaner.run(once=True) if self.multi_vo: reaper.run(once=True, include_rses='vo=%s&(%s)' % (self.vo['vo'], rse_mock), greedy=True) else: reaper.run(once=True, include_rses=rse_mock, greedy=True) abacus_rse.run(once=True) out = api_rse.parse_rse_expression( '%s|%s' % (self.rse_name, self.rse2_name), **self.vo) assert self.rse_name in out assert self.rse2_name in out assert self.rse_id not in out assert self.rse2_id not in out