def test_agis_endpoints_data_parses_proper_json():
response = requests.Response()
response._content = '{"x": "y"}'
response.status_code = 200
with stubbed(requests.get, lambda _: response):
eq_(dumper.agis_endpoints_data(cache=False), {'x': 'y'})
def dump(cls,
subcommand,
ddm_endpoint,
storage_dump,
prev_date_fname=None,
next_date_fname=None,
prev_date=None,
next_date=None,
sort_rucio_replica_dumps=True,
date=None,
cache_dir=DUMPS_CACHE_DIR):
logger = logging.getLogger('auditor.consistency')
if subcommand == 'consistency':
prev_date_fname = data_models.Replica.download(ddm_endpoint,
prev_date,
cache_dir=cache_dir)
next_date_fname = data_models.Replica.download(ddm_endpoint,
next_date,
cache_dir=cache_dir)
assert prev_date_fname is not None
assert next_date_fname is not None
else:
assert subcommand == 'consistency-manual'
prefix = path_parsing.prefix(
dumper.agis_endpoints_data(),
ddm_endpoint,
)
prefix_components = path_parsing.components(prefix)
def parser(line):
'''
Simple parser for Rucio replica dumps.
:param line: String with one line of a dump.
:returns: A tuple with the path and status of the replica.
'''
fields = line.split('\t')
path = fields[6].strip().lstrip('/')
status = fields[8].strip()
return ','.join((path, status))
def strip_storage_dump(line):
'''
Parser to have consistent paths in storage dumps.
:param line: String with one line of a dump.
:returns: Path formated as in the Rucio Replica Dumps.
'''
relative = path_parsing.remove_prefix(
prefix_components,
path_parsing.components(line),
)
if relative[0] == 'rucio':
relative = relative[1:]
return '/'.join(relative)
if sort_rucio_replica_dumps:
prev_date_fname_sorted = gnu_sort(
parse_and_filter_file(prev_date_fname,
parser=parser,
cache_dir=cache_dir),
delimiter=',',
fieldspec='1',
cache_dir=cache_dir,
)
next_date_fname_sorted = gnu_sort(
parse_and_filter_file(next_date_fname,
parser=parser,
cache_dir=cache_dir),
delimiter=',',
fieldspec='1',
cache_dir=cache_dir,
)
else:
prev_date_fname_sorted = parse_and_filter_file(
prev_date_fname,
parser=parser,
cache_dir=cache_dir,
)
next_date_fname_sorted = parse_and_filter_file(
next_date_fname,
parser=parser,
cache_dir=cache_dir,
)
standard_name_re = r'(ddmendpoint_{0}_\d{{2}}-\d{{2}}-\d{{4}}_[0-9a-f]{{40}})$'.format(
ddm_endpoint)
standard_name_match = re.search(standard_name_re, storage_dump)
if standard_name_match is not None:
# If the original filename was generated using the expected format,
# just use the name as prefix for the parsed file.
sd_prefix = standard_name_match.group(0)
elif date is not None:
# Otherwise try to use the date information and DDMEndpoint name to
# have a meaningful filename.
sd_prefix = 'ddmendpoint_{0}_{1}'.format(
ddm_endpoint,
date.strftime('%d-%m-%Y'),
)
else:
# As last resort use only the DDMEndpoint name, but this is error
# prone as old dumps may interfere with the checks.
sd_prefix = 'ddmendpoint_{0}_unknown_date'.format(ddm_endpoint, )
logger.warn(
'Using basic and error prune naming for RSE dump as no date '
'information was provided, %s dump will be named %s',
ddm_endpoint,
sd_prefix,
)
storage_dump_fname_sorted = gnu_sort(
parse_and_filter_file(
storage_dump,
parser=strip_storage_dump,
prefix=sd_prefix,
cache_dir=cache_dir,
),
prefix=sd_prefix,
cache_dir=cache_dir,
)
with open(prev_date_fname_sorted) as prevf:
with open(next_date_fname_sorted) as nextf:
with open(storage_dump_fname_sorted) as sdump:
for path, where, status in compare3(prevf, sdump, nextf):
prevstatus, nextstatus = status
if where[0] and not where[1] and where[2]:
if prevstatus == 'A' and nextstatus == 'A':
yield cls('LOST', path)
if not where[0] and where[1] and not where[2]:
yield cls('DARK', path)
def test_agis_endpoints_data_parses_proper_json(mock_get):
mock_get.return_value = MockResponse(AGISDATA, 200)
assert dumper.agis_endpoints_data(cache=False) == AGISDATA