def test_parse_and_filter_file_default_parameters(self):
''' DUMPER '''
fake_data = 'asd\nasda\n'
path = make_temp_file(self.tmp_dir, fake_data)
parsed_file = parse_and_filter_file(path, cache_dir=self.tmp_dir)
with open(parsed_file) as f:
data = f.read()
eq_(fake_data.replace('\n', '\n\n'), data)
os.unlink(path)
os.unlink(parsed_file)
def test_consistency_manual_transient_file_is_not_lost(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump_1 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tU\n'
rucio_dump_2 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump_1)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump_2)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
eq_(len(list(consistency)), 0)
def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_false_positive(self, mock_get):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
storage_dump = '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
mock_get.return_value = self.fake_agis_data
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
eq_(len(consistency), 0, [e.csv() for e in consistency])
def test_parse_and_filter_file_prefix_and_postfix_specified(self):
''' DUMPER '''
path = make_temp_file(self.tmp_dir, 'x\n')
parsed_file = parse_and_filter_file(path,
prefix=path + 'X',
postfix='Y',
cache_dir=self.tmp_dir)
assert parsed_file == path + 'X_Y'
os.unlink(path)
os.unlink(parsed_file)
def test_parse_and_filter_file_parser_function(self):
''' DUMPER '''
fake_data = 'asd\nasda\n'
path = make_temp_file(self.tmp_dir, fake_data)
parsed_file = parse_and_filter_file(path,
parser=str.strip,
cache_dir=self.tmp_dir)
with open(parsed_file) as f:
data = f.read()
assert fake_data == data
os.unlink(path)
os.unlink(parsed_file)
def test_consistency_manual_lost_file(self):
''' DUMPER '''
rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
rucio_dump += 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n'
storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n'
rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
sdf = make_temp_file(self.tmp_dir, storage_dump)
with stubbed(dumper.agis_endpoints_data, self.fake_agis_data):
consistency = Consistency.dump(
'consistency-manual',
'MOCK_SCRATCHDISK',
sdf,
prev_date_fname=rrdf1,
next_date_fname=rrdf2,
cache_dir=self.tmp_dir,
)
consistency = list(consistency)
eq_(len(consistency), 1)
eq_(consistency[0].apparent_status, 'LOST')
eq_(consistency[0].path, 'user/someuser/aa/bb/user.someuser.filename2')
def test_parse_and_filter_file_filter_function(self):
''' DUMPER '''
fake_data = 'asd\nasda\n'
path = make_temp_file(self.tmp_dir, fake_data)
parsed_file = parse_and_filter_file(path,
filter_=lambda s: s == 'asd\n',
cache_dir=self.tmp_dir)
with open(parsed_file) as f:
data = f.read()
eq_('asd\n\n', data)
os.unlink(path)
os.unlink(parsed_file)
def test_gnu_sort_can_sort_by_field(self):
''' DUMPER '''
unsorted_data = ''.join(['1,z\n', '2,a\n', '3,\xc3\xb1\n'])
sorted_data = ''.join(['2,a\n', '1,z\n', '3,\xc3\xb1\n'])
path = make_temp_file(self.tmp_dir, unsorted_data)
sorted_file = gnu_sort(path,
delimiter=',',
fieldspec='2',
cache_dir=self.tmp_dir)
with open(sorted_file) as f:
eq_(f.read(), sorted_data)
os.unlink(path)
os.unlink(sorted_file)
def test_gnu_sort_can_sort_by_field(self):
''' DUMPER '''
unsorted_data = ''.join(['1,z\n', '2,a\n', '3,\xc3\xb1\n'])
sorted_data = ''.join(['2,a\n', '1,z\n', '3,\xc3\xb1\n'])
path = make_temp_file(self.tmp_dir, unsorted_data)
sorted_file = gnu_sort(path,
delimiter=',',
fieldspec='2',
cache_dir=self.tmp_dir)
if PY3:
with open(sorted_file, encoding='utf-8') as f:
assert f.read() == sorted_data
else:
with open(sorted_file) as f:
assert f.read() == sorted_data
os.unlink(path)
os.unlink(sorted_file)
def test_consistency(self, mock_dumper_get, mock_request_get,
mock_request_head):
''' DUMPER '''
storage_dump = (
'//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.filename\n'
'//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.dark\n')
sd = make_temp_file(self.tmp_dir, storage_dump)
consistency = Consistency.dump('consistency',
'MOCK_SCRATCHDISK',
storage_dump=sd,
prev_date=datetime(2015, 9, 29),
next_date=datetime(2015, 10, 4),
cache_dir=self.tmp_dir)
consistency = list(consistency)
assert len(consistency) == 2
dark = next(entry.path for entry in consistency
if entry.apparent_status == 'DARK')
lost = next(entry.path for entry in consistency
if entry.apparent_status == 'LOST')
assert 'user.someuser.dark' in dark
assert 'user.someuser.lost' in lost
def test_gnu_sort_and_the_current_version_of_python_sort_strings_using_byte_value(self):
''' DUMPER '''
unsorted_data_list = ['z\n', 'a\n', '\xc3\xb1\n']
unsorted_data = ''.join(unsorted_data_list)
sorted_data = ''.join(['a\n', 'z\n', '\xc3\xb1\n'])
path = make_temp_file(self.tmp_dir, unsorted_data)
sorted_file = gnu_sort(path, cache_dir=self.tmp_dir)
if PY3:
with open(sorted_file, encoding='utf-8') as f:
eq_(
f.read(),
sorted_data,
'GNU Sort must sort comparing byte by byte (export '
'LC_ALL=C) to be faster and consistent with Python 2.'
)
else:
with open(sorted_file) as f:
eq_(
f.read(),
sorted_data,
'GNU Sort must sort comparing byte by byte (export '
'LC_ALL=C) to be faster and consistent with Python 2.'
)
os.unlink(path)
os.unlink(sorted_file)
python_sort = ''.join(sorted(unsorted_data_list))
eq_(
python_sort,
sorted_data,
'Current Python interpreter must sort strings comparing byte by '
'byte, it is important to use the same ordering as the one used '
'with GNU Sort. Note Python 3 uses unicode by default.'
)
def test_smart_open_for_text_file():
tmp = make_temp_file('/tmp', 'abcdef')
assert hasattr(dumper.smart_open(tmp), 'read') # check if object is file - python2/3 compatibility
os.unlink(tmp)
def test_smart_open_for_text_file():
tmp = make_temp_file('/tmp', 'abcdef')
ok_(isinstance(dumper.smart_open(tmp), __builtin__.file))
os.unlink(tmp)
