def check_protocols(rse, test_data): rse_id = get_rse_id(rse=rse) protocols = get_rse_protocols(rse_id) assert_equal( test_data[rse]['lfn2pfn_algorithm'], get_rse_attribute('lfn2pfn_algorithm', rse_id=rse_id, use_cache=False)[0]) assert_equal( test_data[rse]['verify_checksum'], get_rse_attribute('verify_checksum', rse_id=rse_id, use_cache=False)[0]) assert_equal(test_data[rse]['availability_write'], protocols['availability_write']) assert_equal(test_data[rse]['availability_read'], protocols['availability_read']) assert_equal(test_data[rse]['availability_delete'], protocols['availability_delete']) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in protocols['protocols']] for protocol in test_data[rse]['protocols']: assert_true({ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol.get('impl', ''), 'prefix': protocol.get('prefix', '') } in protocols)
def get_rses_to_hostname_mapping(): """ Return a dictionaries mapping the RSEs to the hostname of the SE :returns: Dictionary with RSE_id as key and (hostname, rse_info) as value """ result = REGION.get('rse_hostname_mapping') if result is NO_VALUE: result = {} all_rses = list_rses() for rse in all_rses: try: rse_protocol = get_rse_protocols(rse_id=rse['id']) except RSENotFound: logging.log( logging.WARNING, 'RSE deleted while constructing rse-to-hostname mapping. Skipping %s', rse['rse']) continue for prot in rse_protocol['protocols']: if prot['domains']['wan']['delete'] == 1: result[rse['id']] = (prot['hostname'], rse_protocol) if rse['id'] not in result: logging.log(logging.WARNING, 'No default delete protocol for %s', rse['rse']) REGION.set('rse_hostname_mapping', result) return result return result
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None): if rse_id is None: # This can be called directly by client tools if they're co-located on a server # i.e. running rucio cli on a server and during the test suite. # We have to map to VO name here for this situations, despite this nominally # not being a client interface. rse_id = get_rse_id(rse=rse, vo=map_vo(vo)) return get_rse_protocols(rse_id=rse_id, session=session)
def test_importer_core(self): """ IMPORTER (CORE): test import. """ import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) # one protocol should be created, one should be updated check_protocols(self.old_rse_1, self.data1['rses']) # one protocol should be removed as it is not specified in the import data protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) # RSE 4 should be flagged as deleted as it is missing in the import data with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_data(data=self.data2) import_data(data=self.data3)
def get_rse_protocols(rse, issuer): """ Returns all matching protocols (including detailed information) for the given RSE. :param rse: The RSE name. :param issuer: The issuer account. :returns: A dict with all supported protocols and their attibutes. """ return rse_module.get_rse_protocols(rse)
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=deepcopy(self.data1)) # RSE that had not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) new_rse_id = get_rse_id(rse=self.new_rse) protocols = get_rse_protocols(self.old_rse_id_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in protocols['protocols']] assert_true({ 'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3' } not in protocols) attributes = list_rse_attributes(rse_id=new_rse_id) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse_id=new_rse_id) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already exists check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse_id=self.old_rse_id_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse_id=self.old_rse_id_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) self.check_accounts(self.data1['accounts']) with assert_raises(RSENotFound): get_rse(rse_id=self.old_rse_id_4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)
def get_rse(rse): """ Provides details about the specified RSE. :param rse: The RSE name. :param issuer: The issuer account. :returns: a dict with details about the RSE :raises RSENotFound: if the referred RSE was ot found in the database """ return rse_module.get_rse_protocols(rse)
def get_rse_protocols(rse, issuer, vo='def'): """ Returns all matching protocols (including detailed information) for the given RSE. :param rse: The RSE name. :param issuer: The issuer account. :param vo: The VO to act on. :returns: A dict with all supported protocols and their attibutes. """ rse_id = rse_module.get_rse_id(rse=rse, vo=vo) return rse_module.get_rse_protocols(rse_id)
def get_rse(rse): """ Provides details about the specified RSE. :param rse: The RSE name. :param issuer: The issuer account. :returns: a dict with details about the RSE :raises RSENotFound: if the referred RSE was not found in the database """ return rse_module.get_rse_protocols(rse)
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = {'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******'} r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token)} r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before check_rse(self.new_rse, self.data1['rses']) check_protocols(self.new_rse, self.data1['rses']) protocols = get_rse_protocols(self.old_rse_1) protocols = [{'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port']} for protocol in protocols['protocols']] assert_true({'hostename': 'hostname3', 'port': 1000, 'scheme': 'scheme3'} not in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['MinFreeSpace'], 20000) # RSE 1 that already existed before check_rse(self.old_rse_1, self.data1['rses']) check_protocols(self.old_rse_1, self.data1['rses']) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['MaxBeingDeletedFiles'], 1000) assert_equal(limits['MinFreeSpace'], 10000) distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) with assert_raises(RSENotFound): get_rse(self.old_rse_3) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post('/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)
def get_rse(rse, vo='def'): """ Provides details about the specified RSE. :param rse: The RSE name. :param vo: The VO to act on. :returns: a dict with details about the RSE :raises RSENotFound: if the referred RSE was not found in the database """ rse_id = rse_module.get_rse_id(rse=rse, vo=vo) return rse_module.get_rse_protocols(rse_id=rse_id)
def get_authfile_prefixes(self): session = get_session() session.connection() prefixes = set() for rse in list_rses(): rse_id = rse['id'] protocols = get_rse_protocols(rse_id) if 'protocols' in protocols: for protocol in protocols['protocols']: scheme = protocol['scheme'] hostname = protocol['hostname'] port = protocol['port'] prefix = self.clean_prefix_path(protocol['prefix']) authfile_prefix = f"/{scheme}:/{hostname}:{port}/{prefix}" prefixes.add(authfile_prefix) return list(prefixes)
def check_protocols(rse, test_data): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): vo = {'vo': 'tst'} else: vo = {} rse_id = get_rse_id(rse=rse, **vo) protocols = get_rse_protocols(rse_id) assert_equal( test_data[rse]['lfn2pfn_algorithm'], get_rse_attribute('lfn2pfn_algorithm', rse_id=rse_id, use_cache=False)[0]) assert_equal( test_data[rse]['verify_checksum'], get_rse_attribute('verify_checksum', rse_id=rse_id, use_cache=False)[0]) assert_equal(test_data[rse]['availability_write'], protocols['availability_write']) assert_equal(test_data[rse]['availability_read'], protocols['availability_read']) assert_equal(test_data[rse]['availability_delete'], protocols['availability_delete']) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in protocols['protocols']] for protocol in test_data[rse]['protocols']: assert_true({ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol.get('impl', ''), 'prefix': protocol.get('prefix', '') } in protocols)
def get_rses_to_hostname_mapping(): """ Return a dictionaries mapping the RSEs to the hostname of the SE :returns: Dictionary with RSE_id as key and (hostname, rse_info) as value """ result = REGION.get('rse_hostname_mapping') if result is NO_VALUE: result = {} all_rses = list_rses() for rse in all_rses: rse_protocol = get_rse_protocols(rse_id=rse['id']) for prot in rse_protocol['protocols']: if prot['domains']['wan']['delete'] == 1: result[rse['id']] = (prot['hostname'], rse_protocol) if rse['id'] not in result: logging.warn('No default delete protocol for %s', rse['rse']) REGION.set('rse_hostname_mapping', result) return result return result
def reaper(rses, worker_number=0, child_number=0, total_children=1, chunk_size=100, once=False, greedy=False, scheme=None, delay_seconds=0): """ Main loop to select and delete files. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param worker_number: The worker number. :param child_number: The child number. :param total_children: The total number of children created per worker. :param chunk_size: the size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param scheme: Force the reaper to use a particular protocol, e.g., mock. """ logging.info('Starting Reaper: Worker %(worker_number)s, ' 'child %(child_number)s will work on RSEs: ' % locals() + ', '.join([rse['rse'] for rse in rses])) pid = os.getpid() thread = threading.current_thread() hostname = socket.gethostname() executable = ' '.join(sys.argv) # Generate a hash just for the subset of RSEs rse_names = [rse['rse'] for rse in rses] hash_executable = hashlib.sha256((sys.argv[0] + ''.join(rse_names)).encode()).hexdigest() sanity_check(executable=None, hostname=hostname) nothing_to_do = {} while not GRACEFUL_STOP.is_set(): try: # heartbeat heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable) checkpoint_time = datetime.datetime.now() # logging.info('Reaper({0[worker_number]}/{0[child_number]}): Live gives {0[heartbeat]}'.format(locals())) max_deleting_rate = 0 for rse in sort_rses(rses): try: if checkpoint_time + datetime.timedelta(minutes=1) < datetime.datetime.now(): heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable) # logging.info('Reaper({0[worker_number]}/{0[child_number]}): Live gives {0[heartbeat]}'.format(locals())) checkpoint_time = datetime.datetime.now() if rse['id'] in nothing_to_do and nothing_to_do[rse['id']] > datetime.datetime.now(): continue logging.info('Reaper %s-%s: Running on RSE %s %s', worker_number, child_number, rse['rse'], nothing_to_do.get(rse['id'])) rse_info = rsemgr.get_rse_info(rse_id=rse['id']) rse_protocol = rse_core.get_rse_protocols(rse_id=rse['id']) if not rse_protocol['availability_delete']: logging.info('Reaper %s-%s: RSE %s is not available for deletion', worker_number, child_number, rse_info['rse']) nothing_to_do[rse['id']] = datetime.datetime.now() + datetime.timedelta(minutes=30) continue # Temporary hack to force gfal for deletion for protocol in rse_info['protocols']: if protocol['impl'] == 'rucio.rse.protocols.srm.Default' or protocol['impl'] == 'rucio.rse.protocols.gsiftp.Default': protocol['impl'] = 'rucio.rse.protocols.gfal.Default' needed_free_space, max_being_deleted_files = None, 100 needed_free_space_per_child = None if not greedy: max_being_deleted_files, needed_free_space, used, free = __check_rse_usage(rse_id=rse['id']) logging.info('Reaper %(worker_number)s-%(child_number)s: Space usage for RSE %(rse)s - max_being_deleted_files: %(max_being_deleted_files)s, needed_free_space: %(needed_free_space)s, used: %(used)s, free: %(free)s' % locals()) if needed_free_space <= 0: needed_free_space, needed_free_space_per_child = 0, 0 logging.info('Reaper %s-%s: free space is above minimum limit for %s', worker_number, child_number, rse['rse']) else: if total_children and total_children > 0: needed_free_space_per_child = needed_free_space / float(total_children) start = time.time() with monitor.record_timer_block('reaper.list_unlocked_replicas'): replicas = list_unlocked_replicas(rse_id=rse['id'], bytes=needed_free_space_per_child, limit=max_being_deleted_files, worker_number=child_number, total_workers=total_children, delay_seconds=delay_seconds) logging.debug('Reaper %s-%s: list_unlocked_replicas on %s for %s bytes in %s seconds: %s replicas', worker_number, child_number, rse['rse'], needed_free_space_per_child, time.time() - start, len(replicas)) if not replicas: nothing_to_do[rse['id']] = datetime.datetime.now() + datetime.timedelta(minutes=30) logging.info('Reaper %s-%s: No replicas to delete %s. The next check will occur at %s', worker_number, child_number, rse['rse'], nothing_to_do[rse['id']]) continue prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme) for files in chunks(replicas, chunk_size): logging.debug('Reaper %s-%s: Running on : %s', worker_number, child_number, str(files)) try: update_replicas_states(replicas=[dict(list(replica.items()) + [('state', ReplicaState.BEING_DELETED), ('rse_id', rse['id'])]) for replica in files], nowait=True) for replica in files: try: replica['pfn'] = str(list(rsemgr.lfns2pfns(rse_settings=rse_info, lfns=[{'scope': replica['scope'].external, 'name': replica['name'], 'path': replica['path']}], operation='delete', scheme=scheme).values())[0]) except (ReplicaUnAvailable, ReplicaNotFound) as error: err_msg = 'Failed to get pfn UNAVAILABLE replica %s:%s on %s with error %s' % (replica['scope'], replica['name'], rse['rse'], str(error)) logging.warning('Reaper %s-%s: %s', worker_number, child_number, err_msg) replica['pfn'] = None monitor.record_counter(counters='reaper.deletion.being_deleted', delta=len(files)) try: deleted_files = [] prot.connect() for replica in files: try: deletion_dict = {'scope': replica['scope'].external, 'name': replica['name'], 'rse': rse_info['rse'], 'rse_id': rse_info['id'], 'file-size': replica['bytes'], 'bytes': replica['bytes'], 'url': replica['pfn'], 'protocol': prot.attributes['scheme']} if replica['scope'].vo != 'def': deletion_dict['vo'] = replica['scope'].vo logging.info('Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']) start = time.time() if rse['staging_area'] or rse['rse'].endswith("STAGING"): logging.warning('Reaper %s-%s: Deletion STAGING of %s:%s as %s on %s, will only delete the catalog and not do physical deletion', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']) else: if replica['pfn']: pfn = replica['pfn'] # sign the URL if necessary if prot.attributes['scheme'] == 'https' and rse_info['sign_url'] is not None: pfn = get_signed_url(rse['id'], rse_info['sign_url'], 'delete', pfn) prot.delete(pfn) else: logging.warning('Reaper %s-%s: Deletion UNAVAILABLE of %s:%s as %s on %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']) monitor.record_timer('daemons.reaper.delete.%s.%s' % (prot.attributes['scheme'], rse['rse']), (time.time() - start) * 1000) duration = time.time() - start deleted_files.append({'scope': replica['scope'], 'name': replica['name']}) deletion_dict['duration'] = duration add_message('deletion-done', deletion_dict) logging.info('Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], duration) except SourceNotFound: err_msg = 'Deletion NOTFOUND of %s:%s as %s on %s' % (replica['scope'], replica['name'], replica['pfn'], rse['rse']) logging.warning(err_msg) deleted_files.append({'scope': replica['scope'], 'name': replica['name']}) if replica['state'] == ReplicaState.AVAILABLE: deletion_dict['reason'] = str(err_msg) add_message('deletion-failed', deletion_dict) except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error: logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(error)) deletion_dict['reason'] = str(error) add_message('deletion-failed', deletion_dict) except Exception as error: logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc())) deletion_dict['reason'] = str(error) add_message('deletion-failed', deletion_dict) except: logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc())) except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error: for replica in files: logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s', worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(error)) payload = {'scope': replica['scope'].external, 'name': replica['name'], 'rse': rse_info['rse'], 'rse_id': rse_info['id'], 'file-size': replica['bytes'], 'bytes': replica['bytes'], 'url': replica['pfn'], 'reason': str(error), 'protocol': prot.attributes['scheme']} if replica['scope'].vo != 'def': deletion_dict['vo'] = replica['scope'].vo add_message('deletion-failed', payload) break finally: prot.close() start = time.time() with monitor.record_timer_block('reaper.delete_replicas'): delete_replicas(rse_id=rse['id'], files=deleted_files) logging.debug('Reaper %s-%s: delete_replicas successes %s %s %s', worker_number, child_number, rse['rse'], len(deleted_files), time.time() - start) monitor.record_counter(counters='reaper.deletion.done', delta=len(deleted_files)) except DatabaseException as error: logging.warning('Reaper %s-%s: DatabaseException %s', worker_number, child_number, str(error)) except UnsupportedOperation as error: logging.warning('Reaper %s-%s: UnsupportedOperation %s', worker_number, child_number, str(error)) except: logging.critical(traceback.format_exc()) except RSENotFound as error: logging.warning('Reaper %s-%s: RSE not found %s', worker_number, child_number, str(error)) except: logging.critical(traceback.format_exc()) if once: break time.sleep(1) except DatabaseException as error: logging.warning('Reaper: %s', str(error)) except: logging.critical(traceback.format_exc()) die(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable) logging.info('Graceful stop requested') logging.info('Graceful stop done') return
def tmp_rse_info(rse, session=None): rse_id = get_rse_id(rse=rse) return get_rse_protocols(rse_id=rse_id, session=session)
def import_data(data, session=None): """ Import data to add and update records in Rucio. :param data: data to be imported as dictionary. :param session: database session in use. """ # RSEs rses = data.get('rses') if rses: for rse in rses: protocols = rse.get('protocols') if protocols: protocols = protocols.get('protocols') del rse['protocols'] rse_name = rse['rse'] del rse['rse'] if not rse_module.rse_exists(rse_name, session=session): rse_module.add_rse(rse_name, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_name, rse, session=session) # Protocols if protocols: old_protocols = rse_module.get_rse_protocols(rse=rse_name, session=session) for protocol in protocols: scheme = protocol.get('scheme') hostname = protocol.get('hostname') port = protocol.get('port') intersection = [ old_protocol for old_protocol in old_protocols['protocols'] if old_protocol['scheme'] == scheme and old_protocol['hostname'] == hostname and old_protocol['port'] == port ] if intersection: del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse=rse_name, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) else: rse_module.add_protocol(rse=rse_name, parameter=protocol, session=session) # Limits limits = rse.get('limits') if limits: old_limits = rse_module.get_rse_limits(rse=rse_name, session=session) for limit in limits: if limit in old_limits: rse_module.delete_rse_limit(rse=rse_name, name=limit, session=session) rse_module.set_rse_limits(rse=rse_name, name=limit, value=limits[limit], session=session) # Transfer limits transfer_limits = rse.get('transfer_limits') if transfer_limits: for limit in transfer_limits: old_transfer_limits = rse_module.get_rse_transfer_limits( rse=rse_name, activity=limit, session=session) if limit in old_transfer_limits: rse_module.delete_rse_transfer_limits(rse=rse_name, activity=limit, session=session) max_transfers = transfer_limits[limit].items( )[0][1]['max_transfers'] rse_module.set_rse_transfer_limits( rse=rse_name, activity=limit, max_transfers=max_transfers, session=session) # Attributes attributes = rse.get('attributes') if attributes: old_attributes = rse_module.list_rse_attributes( rse=rse_name, session=session) for attr in attributes: if attr in old_attributes: rse_module.del_rse_attribute(rse=rse_name, key=attr, session=session) rse_module.add_rse_attribute(rse=rse_name, key=attr, value=attributes[attr], session=session) # Distances distances = data.get('distances') if distances: for src_rse_name in distances: src = rse_module.get_rse_id(src_rse_name, session=session) for dest_rse_name in distances[src_rse_name]: dest = rse_module.get_rse_id(dest_rse_name, session=session) distance = distances[src_rse_name][dest_rse_name] del distance['src_rse_id'] del distance['dest_rse_id'] old_distance = distance_module.get_distances(src_rse_id=src, dest_rse_id=dest, session=session) if old_distance: distance_module.update_distances(src_rse_id=src, dest_rse_id=dest, parameters=distance, session=session) else: distance_module.add_distance( src_rse_id=src, dest_rse_id=dest, ranking=distance.get('ranking'), agis_distance=distance.get('agis_distance'), geoip_distance=distance.get('geoip_distance'), active=distance.get('active'), submitted=distance.get('submitted'), transfer_speed=distance.get('transfer_speed'), finished=distance.get('finished'), failed=distance.get('failed'), session=session)
def reaper(rses=[], worker_number=1, total_workers=1, chunk_size=100, once=False, scheme=None): """ Main loop to select and delete files. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param worker_number: The worker number. :param total_workers: The total number of workers. :param chunk_size: the size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param scheme: Force the reaper to use a particular protocol, e.g., mock. """ logging.info('Starting Light Reaper %s-%s: Will work on RSEs: %s', worker_number, total_workers, str(rses)) pid = os.getpid() thread = threading.current_thread() hostname = socket.gethostname() executable = ' '.join(sys.argv) hash_executable = hashlib.sha256(sys.argv[0] + ''.join(rses)).hexdigest() sanity_check(executable=None, hostname=hostname) while not GRACEFUL_STOP.is_set(): try: # heartbeat heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable) logging.info( 'Light Reaper({0[worker_number]}/{0[total_workers]}): Live gives {0[heartbeat]}' .format(locals())) nothing_to_do = True random.shuffle(rses) for rse in rses: replicas = list_expired_temporary_dids( rse=rse, limit=chunk_size, worker_number=worker_number, total_workers=total_workers) rse_info = rsemgr.get_rse_info(rse) rse_protocol = rse_core.get_rse_protocols(rse) prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme) deleted_replicas = [] try: prot.connect() for replica in replicas: nothing_to_do = False try: # pfn = str(rsemgr.lfns2pfns(rse_settings=rse_info, # lfns=[{'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']}], # operation='delete', scheme=scheme).values()[0]) pfn = 's3://%s%s%s' % (prot.attributes['hostname'], prot.attributes['prefix'], replica['name']) # logging.debug('Light Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse) start = time.time() prot.delete(pfn) duration = time.time() - start logging.info( 'Light Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s in %s seconds', worker_number, total_workers, replica['scope'], replica['name'], pfn, rse, duration) add_message( 'deletion-done', { 'scope': replica['scope'], 'name': replica['name'], 'rse': rse, 'file-size': replica.get('bytes') or 0, 'bytes': replica.get('bytes') or 0, 'url': pfn, 'duration': duration }) deleted_replicas.append(replica) except SourceNotFound: err_msg = 'Light Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % ( worker_number, total_workers, replica['scope'], replica['name'], pfn, rse) logging.warning(err_msg) deleted_replicas.append(replica) except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error: err_msg = 'Light Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % ( worker_number, total_workers, replica['scope'], replica['name'], pfn, rse, str(error)) logging.warning(err_msg) add_message( 'deletion-failed', { 'scope': replica['scope'], 'name': replica['name'], 'rse': rse, 'file-size': replica['bytes'] or 0, 'bytes': replica['bytes'] or 0, 'url': pfn, 'reason': str(error) }) except: logging.critical(traceback.format_exc()) finally: prot.close() delete_temporary_dids(dids=deleted_replicas) if once: break if once: break if nothing_to_do: logging.info( 'Light Reaper %s-%s: Nothing to do. I will sleep for 60s', worker_number, total_workers) time.sleep(60) except DatabaseException, error: logging.warning('Reaper: %s', str(error)) except:
def import_rses(rses, vo='def', session=None): new_rses = [] for rse_name in rses: rse = rses[rse_name] if isinstance(rse.get('rse_type'), string_types): rse['rse_type'] = RSEType.from_string(str(rse['rse_type'])) try: rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, session=session) except RSENotFound: rse_id = rse_module.add_rse(rse=rse_name, vo=vo, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) else: rse_module.update_rse(rse_id=rse_id, parameters=rse, session=session) new_rses.append(rse_id) # Protocols new_protocols = rse.get('protocols') if new_protocols: # update existing, add missing and remove left over protocols old_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in rse_module.get_rse_protocols( rse_id=rse_id, session=session)['protocols']] missing_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } not in old_protocols ] outdated_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } in old_protocols ] new_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in new_protocols] to_be_removed_protocols = [ old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols ] for protocol in outdated_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse_id=rse_id, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) for protocol in missing_protocols: rse_module.add_protocol(rse_id=rse_id, parameter=protocol, session=session) for protocol in to_be_removed_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session) # Limits old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session) for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']: limit = rse.get(limit_name) if limit: if limit_name in old_limits: rse_module.delete_rse_limit(rse_id=rse_id, name=limit_name, session=session) rse_module.set_rse_limits(rse_id=rse_id, name=limit_name, value=limit, session=session) # Attributes attributes = rse.get('attributes', {}) attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm') attributes['verify_checksum'] = rse.get('verify_checksum') old_attributes = rse_module.list_rse_attributes(rse_id=rse_id, session=session) for attr in attributes: value = attributes[attr] if value is not None: if attr in old_attributes: rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session) rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session) # set deleted flag to RSEs that are missing in the import data old_rses = [ old_rse['id'] for old_rse in rse_module.list_rses(session=session) ] for old_rse in old_rses: if old_rse not in new_rses: try: rse_module.del_rse(rse_id=old_rse, session=session) except RSEOperationNotSupported: pass
def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None): if rse_id is None: rse_id = get_rse_id(rse=rse, vo=vo) return get_rse_protocols(rse_id=rse_id, session=session)
from dogpile.cache import make_region from rucio.core.rse import list_rses, get_rse_protocols rse_region = make_region().configure( 'dogpile.cache.memcached', # 'dogpile.cache.memory' expiration_time=1, arguments={'url': "127.0.0.1:11211", 'distributed_lock': True}, ) if __name__ == '__main__': for rse in list_rses(): rse_info = get_rse_protocols(rse['rse']) print rse['rse'], rse_info rse_region.set(str(rse['rse']), rse_info) # rse_region.delete(str(rse['rse']))
def import_rses(rses, rse_sync_method='edit', attr_sync_method='edit', protocol_sync_method='edit', vo='def', session=None): new_rses = [] for rse_name in rses: rse = rses[rse_name] if isinstance(rse.get('rse_type'), string_types): rse['rse_type'] = RSEType(rse['rse_type']) if rse_module.rse_exists(rse_name, vo=vo, include_deleted=False, session=session): # RSE exists and is active rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, session=session) selected_rse_properties = { key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES } rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session) elif rse_module.rse_exists(rse_name, vo=vo, include_deleted=True, session=session): # RSE exists but in deleted state # Should only modify the RSE if importer is configured for edit or hard sync if rse_sync_method in ['edit', 'hard']: rse_id = rse_module.get_rse_id(rse=rse_name, vo=vo, include_deleted=True, session=session) rse_module.restore_rse(rse_id, session=session) selected_rse_properties = { key: rse[key] for key in rse if key in rse_module.MUTABLE_RSE_PROPERTIES } rse_module.update_rse(rse_id=rse_id, parameters=selected_rse_properties, session=session) else: # Config is in RSE append only mode, should not modify the disabled RSE continue else: rse_id = rse_module.add_rse(rse=rse_name, vo=vo, deterministic=rse.get('deterministic'), volatile=rse.get('volatile'), city=rse.get('city'), region_code=rse.get('region_code'), country_name=rse.get('country_name'), staging_area=rse.get('staging_area'), continent=rse.get('continent'), time_zone=rse.get('time_zone'), ISP=rse.get('ISP'), rse_type=rse.get('rse_type'), latitude=rse.get('latitude'), longitude=rse.get('longitude'), ASN=rse.get('ASN'), availability=rse.get('availability'), session=session) new_rses.append(rse_id) # Protocols new_protocols = rse.get('protocols') if new_protocols: # update existing, add missing and remove left over protocols old_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in rse_module.get_rse_protocols( rse_id=rse_id, session=session)['protocols']] missing_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } not in old_protocols ] outdated_protocols = [ new_protocol for new_protocol in new_protocols if { 'scheme': new_protocol['scheme'], 'hostname': new_protocol['hostname'], 'port': new_protocol['port'] } in old_protocols ] new_protocols = [{ 'scheme': protocol['scheme'], 'hostname': protocol['hostname'], 'port': protocol['port'] } for protocol in new_protocols] to_be_removed_protocols = [ old_protocol for old_protocol in old_protocols if old_protocol not in new_protocols ] if protocol_sync_method == 'append': outdated_protocols = [] for protocol in outdated_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] del protocol['scheme'] del protocol['hostname'] del protocol['port'] rse_module.update_protocols(rse_id=rse_id, scheme=scheme, data=protocol, hostname=hostname, port=port, session=session) for protocol in missing_protocols: rse_module.add_protocol(rse_id=rse_id, parameter=protocol, session=session) if protocol_sync_method == 'hard': for protocol in to_be_removed_protocols: scheme = protocol['scheme'] port = protocol['port'] hostname = protocol['hostname'] rse_module.del_protocols(rse_id=rse_id, scheme=scheme, port=port, hostname=hostname, session=session) # Limits old_limits = rse_module.get_rse_limits(rse_id=rse_id, session=session) for limit_name in ['MaxBeingDeletedFiles', 'MinFreeSpace']: limit = rse.get(limit_name) if limit: if limit_name in old_limits: rse_module.delete_rse_limits(rse_id=rse_id, name=limit_name, session=session) rse_module.set_rse_limits(rse_id=rse_id, name=limit_name, value=limit, session=session) # Attributes attributes = rse.get('attributes', {}) attributes['lfn2pfn_algorithm'] = rse.get('lfn2pfn_algorithm') attributes['verify_checksum'] = rse.get('verify_checksum') old_attributes = rse_module.list_rse_attributes(rse_id=rse_id, session=session) missing_attributes = [ attribute for attribute in old_attributes if attribute not in attributes ] for attr in attributes: value = attributes[attr] if value is not None: if attr in old_attributes: if attr_sync_method not in ['append']: rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session) rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session) else: rse_module.add_rse_attribute(rse_id=rse_id, key=attr, value=value, session=session) if attr_sync_method == 'hard': for attr in missing_attributes: if attr != rse_name: rse_module.del_rse_attribute(rse_id=rse_id, key=attr, session=session) # set deleted flag to RSEs that are missing in the import data old_rses = [ old_rse['id'] for old_rse in rse_module.list_rses(session=session) ] if rse_sync_method == 'hard': for old_rse in old_rses: if old_rse not in new_rses: try: rse_module.del_rse(rse_id=old_rse, session=session) except RSEOperationNotSupported: pass
def reaper(rses, worker_number=1, child_number=1, total_children=1, chunk_size=100, once=False, greedy=False, scheme=None, exclude_rses=None, delay_seconds=0): """ Main loop to select and delete files. :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs. :param worker_number: The worker number. :param child_number: The child number. :param total_children: The total number of children created per worker. :param chunk_size: the size of chunk for deletion. :param once: If True, only runs one iteration of the main loop. :param greedy: If True, delete right away replicas with tombstone. :param scheme: Force the reaper to use a particular protocol, e.g., mock. :param exclude_rses: RSE expression to exclude RSEs from the Reaper. """ logging.info('Starting reaper: worker %(worker_number)s, child %(child_number)s' % locals()) while not graceful_stop.is_set(): try: max_deleting_rate = 0 for rse in rses: deleting_rate = 0 rse_info = rsemgr.get_rse_info(rse['rse']) rse_protocol = rse_core.get_rse_protocols(rse['rse']) if not rse_protocol['availability_delete']: logging.info('Reaper %s-%s: RSE %s is not available for deletion' % (worker_number, child_number, rse_info['rse'])) continue # Temporary hack to force gfal for deletion for protocol in rse_info['protocols']: if protocol['impl'] == 'rucio.rse.protocols.srm.Default' or protocol['impl'] == 'rucio.rse.protocols.gsiftp.Default': protocol['impl'] = 'rucio.rse.protocols.gfal.Default' logging.info('Reaper %s-%s: Running on RSE %s' % (worker_number, child_number, rse_info['rse'])) try: needed_free_space, max_being_deleted_files = None, 10000 if not greedy: max_being_deleted_files, needed_free_space, used, free = __check_rse_usage(rse=rse['rse'], rse_id=rse['id']) logging.info('Reaper %(worker_number)s-%(child_number)s: Space usage for RSE %(rse)s - max_being_deleted_files: %(max_being_deleted_files)s, needed_free_space: %(needed_free_space)s, used: %(used)s, free: %(free)s' % locals()) if needed_free_space <= 0: needed_free_space = 0 logging.info('Reaper %s-%s: free space is above minimum limit for %s' % (worker_number, child_number, rse['rse'])) s = time.time() with monitor.record_timer_block('reaper.list_unlocked_replicas'): replicas = list_unlocked_replicas(rse=rse['rse'], bytes=needed_free_space, limit=max_being_deleted_files, worker_number=child_number, total_workers=total_children, delay_seconds=delay_seconds) logging.debug('Reaper %s-%s: list_unlocked_replicas %s %s %s' % (worker_number, child_number, rse['rse'], time.time() - s, len(replicas))) if not replicas: logging.info('Reaper %s-%s: nothing to do for %s' % (worker_number, child_number, rse['rse'])) continue p = rsemgr.create_protocol(rse_info, 'delete', scheme=None) for files in chunks(replicas, chunk_size): logging.debug('Reaper %s-%s: Running on : %s' % (worker_number, child_number, str(files))) try: s = time.time() update_replicas_states(replicas=[dict(replica.items() + [('state', ReplicaState.BEING_DELETED), ('rse_id', rse['id'])]) for replica in files]) for replica in files: try: replica['pfn'] = str(rsemgr.lfns2pfns(rse_settings=rse_info, lfns=[{'scope': replica['scope'], 'name': replica['name']}, ], operation='delete').values()[0]) except ReplicaUnAvailable as e: err_msg = 'Failed to get pfn UNAVAILABLE replica %s:%s on %s with error %s' % (replica['scope'], replica['name'], rse['rse'], str(e)) logging.warning('Reaper %s-%s: %s' % (worker_number, child_number, err_msg)) replica['pfn'] = None add_message('deletion-planned', {'scope': replica['scope'], 'name': replica['name'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'rse': rse_info['rse']}) # logging.debug('update_replicas_states %s' % (time.time() - s)) monitor.record_counter(counters='reaper.deletion.being_deleted', delta=len(files)) if not scheme: try: deleted_files = [] p.connect() for replica in files: try: logging.info('Reaper %s-%s: Deletion ATTEMPT of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])) s = time.time() if rse['staging_area'] or rse['rse'].endswith("STAGING"): logging.warning('Reaper %s-%s: Deletion STAGING of %s:%s as %s on %s, will only delete the catalog and not do physical deletion' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])) else: if replica['pfn']: p.delete(replica['pfn']) else: logging.warning('Reaper %s-%s: Deletion UNAVAILABLE of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])) monitor.record_timer('daemons.reaper.delete.%s.%s' % (p.attributes['scheme'], rse['rse']), (time.time()-s)*1000) duration = time.time() - s deleted_files.append({'scope': replica['scope'], 'name': replica['name']}) add_message('deletion-done', {'scope': replica['scope'], 'name': replica['name'], 'rse': rse_info['rse'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'duration': duration}) logging.info('Reaper %s-%s: Deletion SUCCESS of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'])) except SourceNotFound: err_msg = 'Reaper %s-%s: Deletion NOTFOUND of %s:%s as %s on %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse']) logging.warning(err_msg) deleted_files.append({'scope': replica['scope'], 'name': replica['name']}) add_message('deletion-failed', {'scope': replica['scope'], 'name': replica['name'], 'rse': rse_info['rse'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'reason': err_msg}) except (ServiceUnavailable, RSEAccessDenied) as e: logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(e))) add_message('deletion-failed', {'scope': replica['scope'], 'name': replica['name'], 'rse': rse_info['rse'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'reason': str(e)}) except Exception as e: logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc()))) add_message('deletion-failed', {'scope': replica['scope'], 'name': replica['name'], 'rse': rse_info['rse'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'reason': str(e)}) except: logging.critical('Reaper %s-%s: Deletion CRITICAL of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(traceback.format_exc()))) except (ServiceUnavailable, RSEAccessDenied) as e: for replica in files: logging.warning('Reaper %s-%s: Deletion NOACCESS of %s:%s as %s on %s: %s' % (worker_number, child_number, replica['scope'], replica['name'], replica['pfn'], rse['rse'], str(e))) add_message('deletion-failed', {'scope': replica['scope'], 'name': replica['name'], 'rse': rse_info['rse'], 'file-size': replica['bytes'], 'url': replica['pfn'], 'reason': str(e)}) finally: p.close() s = time.time() with monitor.record_timer_block('reaper.delete_replicas'): delete_replicas(rse=rse['rse'], files=deleted_files) logging.debug('Reaper %s-%s: delete_replicas successes %s %s %s' % (worker_number, child_number, rse['rse'], len(deleted_files), time.time() - s)) monitor.record_counter(counters='reaper.deletion.done', delta=len(deleted_files)) deleting_rate += len(deleted_files) except: logging.critical(traceback.format_exc()) deleting_rate = deleting_rate * 1.0 / max_being_deleted_files if deleting_rate > max_deleting_rate: max_deleting_rate = deleting_rate except: logging.critical(traceback.format_exc()) if once: break logging.info(" Reaper %s-%s: max_deleting_rate: %s " % (worker_number, child_number, max_deleting_rate)) sleep_time = int((1 - max_deleting_rate) * 60 + 1) time.sleep(sleep_time) except: logging.critical(traceback.format_exc()) logging.info('Graceful stop requested') logging.info('Graceful stop done')
def ddmendpoint_preferred_protocol(ddmendpoint): return next( p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)
def test_importer_client(self): """ IMPORTER (CLIENT): test import. """ import_client = ImportClient() import_client.import_data(data=self.data1) # RSE that had not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already exists rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) import_client.import_data(data=self.data2) import_client.import_data(data=self.data3)
def test_importer_rest(self): """ IMPORTER (REST): test import. """ mw = [] headers1 = { 'X-Rucio-Account': 'root', 'X-Rucio-Username': '******', 'X-Rucio-Password': '******' } r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=headers1, expect_errors=True) token = str(r1.header('X-Rucio-Auth-Token')) headers2 = { 'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root', 'X-Rucio-Auth-Token': str(token) } r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data1)) assert_equal(r2.status, 201) # RSE that not existed before rse = get_rse(self.new_rse) assert_equal(rse['availability'], 5) assert_equal(rse['city'], 'NewCity') assert_equal(rse['rse_type'], RSEType.TAPE) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'] } for protocol in get_rse_protocols(self.new_rse)['protocols']] assert_true({ 'scheme': 'scheme', 'hostname': 'hostname', 'port': 1000 } in protocols) attributes = list_rse_attributes(rse=self.new_rse) assert_equal(attributes['attr1'], 'test') limits = get_rse_limits(rse=self.new_rse) assert_equal(limits['limit1'], 0) transfer_limits = get_rse_transfer_limits(rse=self.new_rse) assert_equal( transfer_limits['activity1'][get_rse_id( self.new_rse)]['max_transfers'], 1) # RSE 1 that already existed before rse = get_rse(self.old_rse_1) assert_equal(rse['rse'], self.old_rse_1) protocols = [{ 'hostname': protocol['hostname'], 'scheme': protocol['scheme'], 'port': protocol['port'], 'impl': protocol['impl'], 'prefix': protocol['prefix'] } for protocol in get_rse_protocols(self.old_rse_1)['protocols']] assert_true({ 'scheme': 'scheme1', 'hostname': 'hostname1', 'port': 1000, 'prefix': 'prefix', 'impl': 'impl1' } in protocols) assert_true({ 'scheme': 'scheme2', 'hostname': 'hostname2', 'port': 1001, 'impl': 'impl', 'prefix': '' } in protocols) attributes = list_rse_attributes(rse=self.old_rse_1) assert_equal(attributes['attr1'], 'test1') assert_equal(attributes['attr2'], 'test2') limits = get_rse_limits(rse=self.old_rse_1) assert_equal(limits['limit1'], 0) assert_equal(limits['limit2'], 2) transfer_limits = get_rse_transfer_limits(rse=self.old_rse_1) assert_equal( transfer_limits['activity1'][get_rse_id( self.old_rse_1)]['max_transfers'], 1) assert_equal( transfer_limits['activity2'][get_rse_id( self.old_rse_1)]['max_transfers'], 2) # Distances distance = get_distances(self.old_rse_id_1, self.old_rse_id_2)[0] assert_equal(distance['ranking'], 10) distance = get_distances(self.old_rse_id_1, self.old_rse_id_3)[0] assert_equal(distance['ranking'], 4) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data2)) assert_equal(r2.status, 201) r2 = TestApp(import_app.wsgifunc(*mw)).post( '/', headers=headers2, expect_errors=True, params=render_json(**self.data3)) assert_equal(r2.status, 201)