def test_core_temporary_dids(): """ TMP DATA IDENTIFIERS (CORE): """ temporary_dids = [] rse = 'MOCK' rse_id = get_rse_id(rse=rse) for _ in range(10): temporary_dids.append({'scope': 'mock', 'name': 'object_%s' % generate_uuid(), 'rse_id': rse_id, 'bytes': 1, 'path': None}) add_temporary_dids(dids=temporary_dids, account='root') compose(scope='mock', name='file_%s' % generate_uuid(), rse_id=rse_id, bytes=10, sources=temporary_dids, account='root', md5=None, adler32=None, pfn=None, meta={}, rules=[], parent_scope=None, parent_name=None) dids = list_expired_temporary_dids(rse_id=rse_id, limit=10) rowcount = delete_temporary_dids(dids=dids) assert_equal(rowcount, 10)
def test_core_temporary_dids(): """ TMP DATA IDENTIFIERS (CORE): """ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} else: vo = {} scope = InternalScope('mock', **vo) root = InternalAccount('root', **vo) temporary_dids = [] rse = 'MOCK' rse_id = get_rse_id(rse=rse, **vo) for _ in range(10): temporary_dids.append({'scope': scope, 'name': 'object_%s' % generate_uuid(), 'rse_id': rse_id, 'bytes': 1, 'path': None}) add_temporary_dids(dids=temporary_dids, account=root) compose(scope=scope, name='file_%s' % generate_uuid(), rse_id=rse_id, bytes=10, sources=temporary_dids, account=root, md5=None, adler32=None, pfn=None, meta={}, rules=[], parent_scope=None, parent_name=None) dids = list_expired_temporary_dids(rse_id=rse_id, limit=10) rowcount = delete_temporary_dids(dids=dids) assert rowcount == 10
def reaper(rses=[], worker_number=1, total_workers=1, chunk_size=100, once=False, scheme=None): """ Main loop to select and delete files. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param worker_number: The worker number. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param scheme: Force the reaper to use a particular protocol, e.g., mock. """ logging.info('Starting Light Reaper %s-%s: Will work on RSEs: %s', worker_number, total_workers, str(rses)) pid = os.getpid() thread = threading.current_thread() hostname = socket.gethostname() executable = ' '.join(sys.argv) hash_executable = hashlib.sha256(sys.argv[0] + ''.join(rses)).hexdigest() sanity_check(executable=None, hostname=hostname) while not GRACEFUL_STOP.is_set(): try: # heartbeat heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable) logging.info( 'Light Reaper({0[worker_number]}/{0[total_workers]}): Live gives {0[heartbeat]}' .format(locals())) nothing_to_do = True random.shuffle(rses) for rse in rses: replicas = list_expired_temporary_dids( rse=rse, limit=chunk_size, worker_number=worker_number, total_workers=total_workers) rse_info = rsemgr.get_rse_info(rse) rse_protocol = rse_core.get_rse_protocols(rse) prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme) deleted_replicas = [] try: prot.connect() for replica in replicas: nothing_to_do = False try: # pfn = str(rsemgr.lfns2pfns(rse_settings=rse_info, # lfns=[{'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']}], # operation='delete', scheme=scheme).values()[0]) pfn = 's3://%s%s%s' % (prot.attributes['hostname'], prot.attributes['prefix'], replica['name']) # logging.debug('Light Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse) start = time.time() prot.delete(pfn) duration = time.time() - start logging.info( 'Light Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse, duration) add_message( 'deletion-done', { 'scope': replica['scope'], 'name': replica['name'], 'rse': rse, 'file-size': replica.get('bytes') or 0, 'bytes': replica.get('bytes') or 0, 'url': pfn, 'duration': duration }) deleted_replicas.append(replica) except SourceNotFound: err_msg = 'Light Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % ( worker_number, total_workers, replica['scope'], replica['name'], pfn, rse) logging.warning(err_msg) deleted_replicas.append(replica) except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error: err_msg = 'Light Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % ( worker_number, total_workers, replica['scope'], replica['name'], pfn, rse, str(error)) logging.warning(err_msg) add_message( 'deletion-failed', { 'scope': replica['scope'], 'name': replica['name'], 'rse': rse, 'file-size': replica['bytes'] or 0, 'bytes': replica['bytes'] or 0, 'url': pfn, 'reason': str(error) }) except: logging.critical(traceback.format_exc()) finally: prot.close() delete_temporary_dids(dids=deleted_replicas) if once: break if once: break if nothing_to_do: logging.info( 'Light Reaper %s-%s: Nothing to do. I will sleep for 60s', worker_number, total_workers) time.sleep(60) except DatabaseException, error: logging.warning('Reaper: %s', str(error)) except: