def consistency(rse, delta, configuration, cache_dir, results_dir):
logger = logging.getLogger('auditor-worker')
rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
results_path = os.path.join(results_dir, '{0}_{1}'.format(rse, rsedate.strftime('%Y%m%d'))) # pylint: disable=no-member
if os.path.exists(results_path + '.bz2') or os.path.exists(results_path):
logger.warning('Consistency check for "%s" (dump dated %s) already done, skipping check', rse, rsedate.strftime('%Y%m%d')) # pylint: disable=no-member
return None
rrdump_prev = ReplicaFromHDFS.download(rse, rsedate - delta, cache_dir=cache_dir)
rrdump_next = ReplicaFromHDFS.download(rse, rsedate + delta, cache_dir=cache_dir)
results = Consistency.dump(
'consistency-manual',
rse,
rsedump,
rrdump_prev,
rrdump_next,
date=rsedate,
cache_dir=cache_dir,
)
mkdir(results_dir)
with temp_file(results_dir, results_path) as (output, _):
for result in results:
output.write('{0}\n'.format(result.csv()))
return results_path
def test_consistency(self, mock_dumper_get, mock_request_get, mock_request_head):
''' DUMPER '''
storage_dump = (
'/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
'/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.dark\n'
)
sd = make_temp_file(self.tmp_dir, storage_dump)
agisdata = [{
'name': 'MOCK_SCRATCHDISK',
'se': 'srm://example.com/',
'endpoint': 'pnfs/example.com/atlas/atlasdatadisk/',
}]
mock_dumper_get.return_value = agisdata
consistency = Consistency.dump('consistency',
'MOCK_SCRATCHDISK',
storage_dump=sd,
prev_date=datetime(2015, 9, 29),
next_date=datetime(2015, 10, 4),
cache_dir=self.tmp_dir)
consistency = list(consistency)
eq_(len(consistency), 2)
dark = next(
entry.path for entry in consistency if entry.apparent_status == 'DARK'
)
lost = next(
entry.path for entry in consistency if entry.apparent_status == 'LOST'
)
ok_('user.someuser.dark' in dark)
ok_('user.someuser.lost' in lost)
def test_consistency_manual_lost_file(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump += 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = RSEPROTOCOL
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
assert len(consistency) == 1
assert consistency[0].apparent_status == 'LOST'
assert consistency[
0].path == 'user/someuser/aa/bb/user.someuser.filename2'
def test_consistency_manual_dark_file(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
storage_dump += 'user/someuser/aa/bb/user.someuser.filename2\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
eq_(len(consistency), 1)
eq_(consistency[0].apparent_status, 'DARK')
eq_(consistency[0].path, 'user/someuser/aa/bb/user.someuser.filename2')
def test_consistency(self):
''' DUMPER '''
rucio_dump_1 = (
'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.lost\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.lost\t2015-09-20 21:22:17\tA\n'
)
rucio_dump_2 = (
'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.lost\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.lost\t2015-09-20 21:22:17\tA\n'
)
storage_dump = (
'/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
'/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.dark\n'
)
sd = make_temp_file(self.tmp_dir, storage_dump)
def fake_get(slf, url, stream=False):
response = requests.Response()
response.status_code = 200
if '29-09-2015' in url:
response.iter_content = lambda _: [rucio_dump_1]
else:
response.iter_content = lambda _: [rucio_dump_2]
return response
def fake_head(slf, url):
response = requests.Response()
response.status_code = 200
return response
agisdata = [{
'name': 'MOCK_SCRATCHDISK',
'se': 'srm://example.com/',
'endpoint': 'pnfs/example.com/atlas/atlasdatadisk/',
}]
with stubbed(dumper.agis_endpoints_data, lambda: agisdata):
with stubbed(requests.Session.get, fake_get):
with stubbed(requests.Session.head, fake_head):
consistency = Consistency.dump(
'consistency',
'MOCK_SCRATCHDISK',
storage_dump=sd,
prev_date=datetime(2015, 9, 29),
next_date=datetime(2015, 10, 4),
cache_dir=self.tmp_dir)
consistency = list(consistency)
eq_(len(consistency), 2)
dark = next(entry.path for entry in consistency
if entry.apparent_status == 'DARK')
lost = next(entry.path for entry in consistency
if entry.apparent_status == 'LOST')
ok_('user.someuser.dark' in dark)
ok_('user.someuser.lost' in lost)
def test_consistency_manual_correct_file_default_args(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
eq_(len(list(consistency)), 0)
def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_false_positive(
self):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
storage_dump = '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
with stubbed(dumper.agis_endpoints_data, self.fake_agis_data):
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
eq_(len(consistency), 0, [e.csv() for e in consistency])
def test_consistency_manual_transient_file_is_not_lost(self):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump_1 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tU\n'
rucio_dump_2 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump_1)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump_2)
sdf = make_temp_file(self.tmp_dir, storage_dump)
with stubbed(dumper.agis_endpoints_data, self.fake_agis_data):
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
eq_(len(list(consistency)), 0)
def test_consistency_manual_transient_file_is_not_lost(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump_1 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tU\n'
rucio_dump_2 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump_1)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump_2)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
assert len(list(consistency)) == 0
def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_false_positive(
self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
storage_dump = '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
assert len(consistency) == 0
def test_consistency_manual_lost_file(self):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump += 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
with stubbed(dumper.agis_endpoints_data, self.fake_agis_data):
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
eq_(len(consistency), 1)
eq_(consistency[0].apparent_status, 'LOST')
eq_(consistency[0].path, 'user/someuser/aa/bb/user.someuser.filename2')
def test_consistency(self, mock_dumper_get, mock_request_get,
mock_request_head):
''' DUMPER '''
storage_dump = (
'//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.filename\n'
'//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.dark\n')
sd = make_temp_file(self.tmp_dir, storage_dump)
consistency = Consistency.dump('consistency',
'MOCK_SCRATCHDISK',
storage_dump=sd,
prev_date=datetime(2015, 9, 29),
next_date=datetime(2015, 10, 4),
cache_dir=self.tmp_dir)
consistency = list(consistency)
assert len(consistency) == 2
dark = next(entry.path for entry in consistency
if entry.apparent_status == 'DARK')
lost = next(entry.path for entry in consistency
if entry.apparent_status == 'LOST')
assert 'user.someuser.dark' in dark
assert 'user.someuser.lost' in lost