def test_ddmendpoint_url_builds_url_from_agis_records():
agisdata = [{
'name': 'SOMEENDPOINT',
'se': 'srm://example.com/',
'endpoint': 'atlasdatadisk/',
}]
with stubbed(dumper.agis_endpoints_data, lambda: agisdata):
eq_(dumper.ddmendpoint_url('SOMEENDPOINT'), 'srm://example.com/atlasdatadisk/')
def test_ddmendpoint_url_builds_url_from_agis_records():
agisdata = [{
'arprotocols': {
'read_wan': [{
'endpoint': 'srm://example.com',
'path': '/atlasdatadisk/rucio/'
}]
},
'name': 'SOMEENDPOINT',
}]
with stubbed(dumper.agis_endpoints_data, lambda: agisdata):
eq_(dumper.ddmendpoint_url('SOMEENDPOINT'),
'srm://example.com/atlasdatadisk/')
def generate_url(rse, config):
'''
:param rse: Name of the endpoint.
:param config: RawConfigParser instance which may have configuration
related to the endpoint.
:returns: Tuple with the URL where the links can be queried to find new
dumps and the pattern used to parse the date of the dump of the files/directories
listed..
'''
site = rse.split('_')[0]
if site not in config.sections():
base_url = ddmendpoint_url(rse) + 'dumps'
url_pattern = 'dump_%Y%m%d'
else:
url_components = config.get(site, rse).split('/')
# The pattern may not be the last component
pattern_index = next(idx for idx, comp in enumerate(url_components) if '%m' in comp)
base_url = '/'.join(url_components[:pattern_index])
url_pattern = '/'.join(url_components[pattern_index:])
return base_url, url_pattern
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get):
mock_get.return_value = []
with pytest.raises(StopIteration):
dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_builds_url_from_agis_records(mock_get):
mock_get.return_value = AGISDATA
assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'srm://example.com/atlasdatadisk/'
def test_ddmendpoint_url_fails_on_unexistent_entry():
with stubbed(dumper.agis_endpoints_data, lambda: []):
dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get):
mock_get.side_effect = StopIteration()
with pytest.raises(StopIteration):
dumper.ddmendpoint_url('SOMEENDPOINT')
def test_ddmendpoint_url_builds_url_from_ddmendpoint_preferred_protocol(mock_get):
mock_get.return_value = RSEPROTOCOL
assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'root://example.com:1094//atlasdatadisk/'
def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get):
mock_get.return_value = []
dumper.ddmendpoint_url('SOMEENDPOINT')
def dump(cls,
subcommand,
ddm_endpoint,
storage_dump,
prev_date_fname=None,
next_date_fname=None,
prev_date=None,
next_date=None,
sort_rucio_replica_dumps=True,
date=None,
cache_dir=DUMPS_CACHE_DIR):
logger = logging.getLogger('auditor.consistency')
if subcommand == 'consistency':
prev_date_fname = data_models.Replica.download(ddm_endpoint,
prev_date,
cache_dir=cache_dir)
next_date_fname = data_models.Replica.download(ddm_endpoint,
next_date,
cache_dir=cache_dir)
assert prev_date_fname is not None
assert next_date_fname is not None
else:
assert subcommand == 'consistency-manual'
prefix_components = path_parsing.components(
dumper.ddmendpoint_url(ddm_endpoint))
def parser(line):
'''
Simple parser for Rucio replica dumps.
:param line: String with one line of a dump.
:returns: A tuple with the path and status of the replica.
'''
fields = line.split('\t')
path = fields[6].strip().lstrip('/')
status = fields[8].strip()
return ','.join((path, status))
def strip_storage_dump(line):
'''
Parser to have consistent paths in storage dumps.
:param line: String with one line of a dump.
:returns: Path formated as in the Rucio Replica Dumps.
'''
relative = path_parsing.remove_prefix(
prefix_components,
path_parsing.components(line),
)
if relative[0] == 'rucio':
relative = relative[1:]
return '/'.join(relative)
if sort_rucio_replica_dumps:
prev_date_fname_sorted = gnu_sort(
parse_and_filter_file(prev_date_fname,
parser=parser,
cache_dir=cache_dir),
delimiter=',',
fieldspec='1',
cache_dir=cache_dir,
)
next_date_fname_sorted = gnu_sort(
parse_and_filter_file(next_date_fname,
parser=parser,
cache_dir=cache_dir),
delimiter=',',
fieldspec='1',
cache_dir=cache_dir,
)
else:
prev_date_fname_sorted = parse_and_filter_file(
prev_date_fname,
parser=parser,
cache_dir=cache_dir,
)
next_date_fname_sorted = parse_and_filter_file(
next_date_fname,
parser=parser,
cache_dir=cache_dir,
)
standard_name_re = r'(ddmendpoint_{0}_\d{{2}}-\d{{2}}-\d{{4}}_[0-9a-f]{{40}})$'.format(
ddm_endpoint)
standard_name_match = re.search(standard_name_re, storage_dump)
if standard_name_match is not None:
# If the original filename was generated using the expected format,
# just use the name as prefix for the parsed file.
sd_prefix = standard_name_match.group(0)
elif date is not None:
# Otherwise try to use the date information and DDMEndpoint name to
# have a meaningful filename.
sd_prefix = 'ddmendpoint_{0}_{1}'.format(
ddm_endpoint,
date.strftime('%d-%m-%Y'),
)
else:
# As last resort use only the DDMEndpoint name, but this is error
# prone as old dumps may interfere with the checks.
sd_prefix = 'ddmendpoint_{0}_unknown_date'.format(ddm_endpoint, )
logger.warning(
'Using basic and error prune naming for RSE dump as no date '
'information was provided, %s dump will be named %s',
ddm_endpoint,
sd_prefix,
)
storage_dump_fname_sorted = gnu_sort(
parse_and_filter_file(
storage_dump,
parser=strip_storage_dump,
prefix=sd_prefix,
cache_dir=cache_dir,
),
prefix=sd_prefix,
cache_dir=cache_dir,
)
with open(prev_date_fname_sorted) as prevf:
with open(next_date_fname_sorted) as nextf:
with open(storage_dump_fname_sorted) as sdump:
for path, where, status in compare3(prevf, sdump, nextf):
prevstatus, nextstatus = status
if where[0] and not where[1] and where[2]:
if prevstatus == 'A' and nextstatus == 'A':
yield cls('LOST', path)
if not where[0] and where[1] and not where[2]:
yield cls('DARK', path)
