def test_ddmendpoint_url_builds_url_from_agis_records(): agisdata = [{ 'name': 'SOMEENDPOINT', 'se': 'srm://example.com/', 'endpoint': 'atlasdatadisk/', }] with stubbed(dumper.agis_endpoints_data, lambda: agisdata): eq_(dumper.ddmendpoint_url('SOMEENDPOINT'), 'srm://example.com/atlasdatadisk/')
def test_ddmendpoint_url_builds_url_from_agis_records(): agisdata = [{ 'arprotocols': { 'read_wan': [{ 'endpoint': 'srm://example.com', 'path': '/atlasdatadisk/rucio/' }] }, 'name': 'SOMEENDPOINT', }] with stubbed(dumper.agis_endpoints_data, lambda: agisdata): eq_(dumper.ddmendpoint_url('SOMEENDPOINT'), 'srm://example.com/atlasdatadisk/')
def generate_url(rse, config): ''' :param rse: Name of the endpoint. :param config: RawConfigParser instance which may have configuration related to the endpoint. :returns: Tuple with the URL where the links can be queried to find new dumps and the pattern used to parse the date of the dump of the files/directories listed.. ''' site = rse.split('_')[0] if site not in config.sections(): base_url = ddmendpoint_url(rse) + 'dumps' url_pattern = 'dump_%Y%m%d' else: url_components = config.get(site, rse).split('/') # The pattern may not be the last component pattern_index = next(idx for idx, comp in enumerate(url_components) if '%m' in comp) base_url = '/'.join(url_components[:pattern_index]) url_pattern = '/'.join(url_components[pattern_index:]) return base_url, url_pattern
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get): mock_get.return_value = [] with pytest.raises(StopIteration): dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_builds_url_from_agis_records(mock_get): mock_get.return_value = AGISDATA assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'srm://example.com/atlasdatadisk/'
def test_ddmendpoint_url_fails_on_unexistent_entry(): with stubbed(dumper.agis_endpoints_data, lambda: []): dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get): mock_get.side_effect = StopIteration() with pytest.raises(StopIteration): dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_builds_url_from_ddmendpoint_preferred_protocol(mock_get): mock_get.return_value = RSEPROTOCOL assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'root://example.com:1094//atlasdatadisk/'
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get): mock_get.return_value = [] dumper.ddmendpoint_url('SOMEENDPOINT')
def dump(cls, subcommand, ddm_endpoint, storage_dump, prev_date_fname=None, next_date_fname=None, prev_date=None, next_date=None, sort_rucio_replica_dumps=True, date=None, cache_dir=DUMPS_CACHE_DIR): logger = logging.getLogger('auditor.consistency') if subcommand == 'consistency': prev_date_fname = data_models.Replica.download(ddm_endpoint, prev_date, cache_dir=cache_dir) next_date_fname = data_models.Replica.download(ddm_endpoint, next_date, cache_dir=cache_dir) assert prev_date_fname is not None assert next_date_fname is not None else: assert subcommand == 'consistency-manual' prefix_components = path_parsing.components( dumper.ddmendpoint_url(ddm_endpoint)) def parser(line): ''' Simple parser for Rucio replica dumps. :param line: String with one line of a dump. :returns: A tuple with the path and status of the replica. ''' fields = line.split('\t') path = fields[6].strip().lstrip('/') status = fields[8].strip() return ','.join((path, status)) def strip_storage_dump(line): ''' Parser to have consistent paths in storage dumps. :param line: String with one line of a dump. :returns: Path formated as in the Rucio Replica Dumps. ''' relative = path_parsing.remove_prefix( prefix_components, path_parsing.components(line), ) if relative[0] == 'rucio': relative = relative[1:] return '/'.join(relative) if sort_rucio_replica_dumps: prev_date_fname_sorted = gnu_sort( parse_and_filter_file(prev_date_fname, parser=parser, cache_dir=cache_dir), delimiter=',', fieldspec='1', cache_dir=cache_dir, ) next_date_fname_sorted = gnu_sort( parse_and_filter_file(next_date_fname, parser=parser, cache_dir=cache_dir), delimiter=',', fieldspec='1', cache_dir=cache_dir, ) else: prev_date_fname_sorted = parse_and_filter_file( prev_date_fname, parser=parser, cache_dir=cache_dir, ) next_date_fname_sorted = parse_and_filter_file( next_date_fname, parser=parser, cache_dir=cache_dir, ) standard_name_re = r'(ddmendpoint_{0}_\d{{2}}-\d{{2}}-\d{{4}}_[0-9a-f]{{40}})$'.format( ddm_endpoint) standard_name_match = re.search(standard_name_re, storage_dump) if standard_name_match is not None: # If the original filename was generated using the expected format, # just use the name as prefix for the parsed file. sd_prefix = standard_name_match.group(0) elif date is not None: # Otherwise try to use the date information and DDMEndpoint name to # have a meaningful filename. sd_prefix = 'ddmendpoint_{0}_{1}'.format( ddm_endpoint, date.strftime('%d-%m-%Y'), ) else: # As last resort use only the DDMEndpoint name, but this is error # prone as old dumps may interfere with the checks. sd_prefix = 'ddmendpoint_{0}_unknown_date'.format(ddm_endpoint, ) logger.warning( 'Using basic and error prune naming for RSE dump as no date ' 'information was provided, %s dump will be named %s', ddm_endpoint, sd_prefix, ) storage_dump_fname_sorted = gnu_sort( parse_and_filter_file( storage_dump, parser=strip_storage_dump, prefix=sd_prefix, cache_dir=cache_dir, ), prefix=sd_prefix, cache_dir=cache_dir, ) with open(prev_date_fname_sorted) as prevf: with open(next_date_fname_sorted) as nextf: with open(storage_dump_fname_sorted) as sdump: for path, where, status in compare3(prevf, sdump, nextf): prevstatus, nextstatus = status if where[0] and not where[1] and where[2]: if prevstatus == 'A' and nextstatus == 'A': yield cls('LOST', path) if not where[0] and where[1] and not where[2]: yield cls('DARK', path)