def test_parse_and_filter_file_default_parameters(self): ''' DUMPER ''' fake_data = 'asd\nasda\n' path = make_temp_file(self.tmp_dir, fake_data) parsed_file = parse_and_filter_file(path, cache_dir=self.tmp_dir) with open(parsed_file) as f: data = f.read() eq_(fake_data.replace('\n', '\n\n'), data) os.unlink(path) os.unlink(parsed_file)
def test_consistency_manual_transient_file_is_not_lost(self, mock_get): ''' DUMPER ''' rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n' rucio_dump_1 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tU\n' rucio_dump_2 = rucio_dump + 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n' storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n' rrdf1 = make_temp_file(self.tmp_dir, rucio_dump_1) rrdf2 = make_temp_file(self.tmp_dir, rucio_dump_2) sdf = make_temp_file(self.tmp_dir, storage_dump) mock_get.return_value = self.fake_agis_data consistency = Consistency.dump( 'consistency-manual', 'MOCK_SCRATCHDISK', sdf, prev_date_fname=rrdf1, next_date_fname=rrdf2, cache_dir=self.tmp_dir, ) eq_(len(list(consistency)), 0)
def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_false_positive(self, mock_get): ''' DUMPER ''' rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n' storage_dump = '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n' rrdf1 = make_temp_file(self.tmp_dir, rucio_dump) rrdf2 = make_temp_file(self.tmp_dir, rucio_dump) sdf = make_temp_file(self.tmp_dir, storage_dump) mock_get.return_value = self.fake_agis_data consistency = Consistency.dump( 'consistency-manual', 'MOCK_SCRATCHDISK', sdf, prev_date_fname=rrdf1, next_date_fname=rrdf2, cache_dir=self.tmp_dir, ) consistency = list(consistency) eq_(len(consistency), 0, [e.csv() for e in consistency])
def test_parse_and_filter_file_prefix_and_postfix_specified(self): ''' DUMPER ''' path = make_temp_file(self.tmp_dir, 'x\n') parsed_file = parse_and_filter_file(path, prefix=path + 'X', postfix='Y', cache_dir=self.tmp_dir) assert parsed_file == path + 'X_Y' os.unlink(path) os.unlink(parsed_file)
def test_parse_and_filter_file_parser_function(self): ''' DUMPER ''' fake_data = 'asd\nasda\n' path = make_temp_file(self.tmp_dir, fake_data) parsed_file = parse_and_filter_file(path, parser=str.strip, cache_dir=self.tmp_dir) with open(parsed_file) as f: data = f.read() assert fake_data == data os.unlink(path) os.unlink(parsed_file)
def test_consistency_manual_lost_file(self): ''' DUMPER ''' rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n' rucio_dump += 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename2\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename2\t2015-09-20 21:22:17\tA\n' storage_dump = 'user/someuser/aa/bb/user.someuser.filename\n' rrdf1 = make_temp_file(self.tmp_dir, rucio_dump) rrdf2 = make_temp_file(self.tmp_dir, rucio_dump) sdf = make_temp_file(self.tmp_dir, storage_dump) with stubbed(dumper.agis_endpoints_data, self.fake_agis_data): consistency = Consistency.dump( 'consistency-manual', 'MOCK_SCRATCHDISK', sdf, prev_date_fname=rrdf1, next_date_fname=rrdf2, cache_dir=self.tmp_dir, ) consistency = list(consistency) eq_(len(consistency), 1) eq_(consistency[0].apparent_status, 'LOST') eq_(consistency[0].path, 'user/someuser/aa/bb/user.someuser.filename2')
def test_parse_and_filter_file_filter_function(self): ''' DUMPER ''' fake_data = 'asd\nasda\n' path = make_temp_file(self.tmp_dir, fake_data) parsed_file = parse_and_filter_file(path, filter_=lambda s: s == 'asd\n', cache_dir=self.tmp_dir) with open(parsed_file) as f: data = f.read() eq_('asd\n\n', data) os.unlink(path) os.unlink(parsed_file)
def test_gnu_sort_can_sort_by_field(self): ''' DUMPER ''' unsorted_data = ''.join(['1,z\n', '2,a\n', '3,\xc3\xb1\n']) sorted_data = ''.join(['2,a\n', '1,z\n', '3,\xc3\xb1\n']) path = make_temp_file(self.tmp_dir, unsorted_data) sorted_file = gnu_sort(path, delimiter=',', fieldspec='2', cache_dir=self.tmp_dir) with open(sorted_file) as f: eq_(f.read(), sorted_data) os.unlink(path) os.unlink(sorted_file)
def test_gnu_sort_can_sort_by_field(self): ''' DUMPER ''' unsorted_data = ''.join(['1,z\n', '2,a\n', '3,\xc3\xb1\n']) sorted_data = ''.join(['2,a\n', '1,z\n', '3,\xc3\xb1\n']) path = make_temp_file(self.tmp_dir, unsorted_data) sorted_file = gnu_sort(path, delimiter=',', fieldspec='2', cache_dir=self.tmp_dir) if PY3: with open(sorted_file, encoding='utf-8') as f: assert f.read() == sorted_data else: with open(sorted_file) as f: assert f.read() == sorted_data os.unlink(path) os.unlink(sorted_file)
def test_consistency(self, mock_dumper_get, mock_request_get, mock_request_head): ''' DUMPER ''' storage_dump = ( '//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.filename\n' '//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.dark\n') sd = make_temp_file(self.tmp_dir, storage_dump) consistency = Consistency.dump('consistency', 'MOCK_SCRATCHDISK', storage_dump=sd, prev_date=datetime(2015, 9, 29), next_date=datetime(2015, 10, 4), cache_dir=self.tmp_dir) consistency = list(consistency) assert len(consistency) == 2 dark = next(entry.path for entry in consistency if entry.apparent_status == 'DARK') lost = next(entry.path for entry in consistency if entry.apparent_status == 'LOST') assert 'user.someuser.dark' in dark assert 'user.someuser.lost' in lost
def test_gnu_sort_and_the_current_version_of_python_sort_strings_using_byte_value(self): ''' DUMPER ''' unsorted_data_list = ['z\n', 'a\n', '\xc3\xb1\n'] unsorted_data = ''.join(unsorted_data_list) sorted_data = ''.join(['a\n', 'z\n', '\xc3\xb1\n']) path = make_temp_file(self.tmp_dir, unsorted_data) sorted_file = gnu_sort(path, cache_dir=self.tmp_dir) if PY3: with open(sorted_file, encoding='utf-8') as f: eq_( f.read(), sorted_data, 'GNU Sort must sort comparing byte by byte (export ' 'LC_ALL=C) to be faster and consistent with Python 2.' ) else: with open(sorted_file) as f: eq_( f.read(), sorted_data, 'GNU Sort must sort comparing byte by byte (export ' 'LC_ALL=C) to be faster and consistent with Python 2.' ) os.unlink(path) os.unlink(sorted_file) python_sort = ''.join(sorted(unsorted_data_list)) eq_( python_sort, sorted_data, 'Current Python interpreter must sort strings comparing byte by ' 'byte, it is important to use the same ordering as the one used ' 'with GNU Sort. Note Python 3 uses unicode by default.' )
def test_smart_open_for_text_file(): tmp = make_temp_file('/tmp', 'abcdef') assert hasattr(dumper.smart_open(tmp), 'read') # check if object is file - python2/3 compatibility os.unlink(tmp)
def test_smart_open_for_text_file(): tmp = make_temp_file('/tmp', 'abcdef') ok_(isinstance(dumper.smart_open(tmp), __builtin__.file)) os.unlink(tmp)