def test_core_temporary_dids():
""" TMP DATA IDENTIFIERS (CORE): """
temporary_dids = []
rse = 'MOCK'
rse_id = get_rse_id(rse=rse)
for _ in range(10):
temporary_dids.append({'scope': 'mock',
'name': 'object_%s' % generate_uuid(),
'rse_id': rse_id,
'bytes': 1,
'path': None})
add_temporary_dids(dids=temporary_dids, account='root')
compose(scope='mock', name='file_%s' % generate_uuid(), rse_id=rse_id,
bytes=10, sources=temporary_dids, account='root',
md5=None, adler32=None, pfn=None, meta={}, rules=[],
parent_scope=None, parent_name=None)
dids = list_expired_temporary_dids(rse_id=rse_id, limit=10)
rowcount = delete_temporary_dids(dids=dids)
assert_equal(rowcount, 10)
def test_core_temporary_dids():
""" TMP DATA IDENTIFIERS (CORE): """
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo = {}
scope = InternalScope('mock', **vo)
root = InternalAccount('root', **vo)
temporary_dids = []
rse = 'MOCK'
rse_id = get_rse_id(rse=rse, **vo)
for _ in range(10):
temporary_dids.append({'scope': scope,
'name': 'object_%s' % generate_uuid(),
'rse_id': rse_id,
'bytes': 1,
'path': None})
add_temporary_dids(dids=temporary_dids, account=root)
compose(scope=scope, name='file_%s' % generate_uuid(), rse_id=rse_id,
bytes=10, sources=temporary_dids, account=root,
md5=None, adler32=None, pfn=None, meta={}, rules=[],
parent_scope=None, parent_name=None)
dids = list_expired_temporary_dids(rse_id=rse_id, limit=10)
rowcount = delete_temporary_dids(dids=dids)
assert rowcount == 10
def reaper(rses=[],
worker_number=1,
total_workers=1,
chunk_size=100,
once=False,
scheme=None):
"""
Main loop to select and delete files.
:param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs.
:param worker_number: The worker number.
:param total_workers: The total number of workers.
:param chunk_size: the size of chunk for deletion.
:param once: If True, only runs one iteration of the main loop.
:param scheme: Force the reaper to use a particular protocol, e.g., mock.
"""
logging.info('Starting Light Reaper %s-%s: Will work on RSEs: %s',
worker_number, total_workers, str(rses))
pid = os.getpid()
thread = threading.current_thread()
hostname = socket.gethostname()
executable = ' '.join(sys.argv)
hash_executable = hashlib.sha256(sys.argv[0] + ''.join(rses)).hexdigest()
sanity_check(executable=None, hostname=hostname)
while not GRACEFUL_STOP.is_set():
try:
# heartbeat
heartbeat = live(executable=executable,
hostname=hostname,
pid=pid,
thread=thread,
hash_executable=hash_executable)
logging.info(
'Light Reaper({0[worker_number]}/{0[total_workers]}): Live gives {0[heartbeat]}'
.format(locals()))
nothing_to_do = True
random.shuffle(rses)
for rse in rses:
replicas = list_expired_temporary_dids(
rse=rse,
limit=chunk_size,
worker_number=worker_number,
total_workers=total_workers)
rse_info = rsemgr.get_rse_info(rse)
rse_protocol = rse_core.get_rse_protocols(rse)
prot = rsemgr.create_protocol(rse_info,
'delete',
scheme=scheme)
deleted_replicas = []
try:
prot.connect()
for replica in replicas:
nothing_to_do = False
try:
# pfn = str(rsemgr.lfns2pfns(rse_settings=rse_info,
# lfns=[{'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']}],
# operation='delete', scheme=scheme).values()[0])
pfn = 's3://%s%s%s' % (prot.attributes['hostname'],
prot.attributes['prefix'],
replica['name'])
# logging.debug('Light Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse)
start = time.time()
prot.delete(pfn)
duration = time.time() - start
logging.info(
'Light Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds',
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse, duration)
add_message(
'deletion-done', {
'scope': replica['scope'],
'name': replica['name'],
'rse': rse,
'file-size': replica.get('bytes') or 0,
'bytes': replica.get('bytes') or 0,
'url': pfn,
'duration': duration
})
deleted_replicas.append(replica)
except SourceNotFound:
err_msg = 'Light Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % (
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse)
logging.warning(err_msg)
deleted_replicas.append(replica)
except (ServiceUnavailable, RSEAccessDenied,
ResourceTemporaryUnavailable) as error:
err_msg = 'Light Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (
worker_number, total_workers, replica['scope'],
replica['name'], pfn, rse, str(error))
logging.warning(err_msg)
add_message(
'deletion-failed', {
'scope': replica['scope'],
'name': replica['name'],
'rse': rse,
'file-size': replica['bytes'] or 0,
'bytes': replica['bytes'] or 0,
'url': pfn,
'reason': str(error)
})
except:
logging.critical(traceback.format_exc())
finally:
prot.close()
delete_temporary_dids(dids=deleted_replicas)
if once:
break
if once:
break
if nothing_to_do:
logging.info(
'Light Reaper %s-%s: Nothing to do. I will sleep for 60s',
worker_number, total_workers)
time.sleep(60)
except DatabaseException, error:
logging.warning('Reaper: %s', str(error))
except: