def test_run_once_recover_from_failure(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect try: object_replicator.http_connect = mock_http_connect(200) # Write some files into '1' and run replicate- they should be moved # to the other partitoins and then node should get deleted. cur_part = '1' df = diskfile.DiskFile( self.devices, 'sda', cur_part, 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue(os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) for i, result in [('0', True), ('1', False), ('2', True), ('3', True)]: self.assertEquals(os.access( os.path.join(self.objects, i, diskfile.HASH_FILE), os.F_OK), result) finally: object_replicator.http_connect = was_connector
def test_get_hashes_unmodified(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) with open( os.path.join(df.datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes = diskfile.get_hashes(part) i = [0] def _getmtime(filename): i[0] += 1 return 1 with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): hashed, hashes = diskfile.get_hashes(part, recalculate=['a83']) self.assertEquals(i[0], 2)
def test_hash_suffix_hash_dir_is_file_quarantine(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(os.path.dirname(df.datadir)) open(df.datadir, 'wb').close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) orig_quarantine_renamer = diskfile.quarantine_renamer called = [False] def wrapped(*args, **kwargs): called[0] = True return orig_quarantine_renamer(*args, **kwargs) try: diskfile.quarantine_renamer = wrapped diskfile.hash_suffix(whole_path_from, 101) finally: diskfile.quarantine_renamer = orig_quarantine_renamer self.assertTrue(called[0])
def test_get_hashes_unmodified_and_zero_bytes(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) part = os.path.join(self.objects, '0') open(os.path.join(part, diskfile.HASH_FILE), 'w') # Now the hash file is zero bytes. i = [0] def _getmtime(filename): i[0] += 1 return 1 with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): hashed, hashes = diskfile.get_hashes(part, recalculate=[]) # getmtime will actually not get called. Initially, the pickle.load # will raise an exception first and later, force_rewrite will # short-circuit the if clause to determine whether to write out a # fresh hashes_file. self.assertEquals(i[0], 0) self.assertTrue('a83' in hashes)
def test_quarantine_same_file(self): df = self._create_test_file('empty') new_dir = df.quarantine() quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(df.data_file))) self.assert_(os.path.isdir(quar_dir)) self.assertEquals(quar_dir, new_dir) # have to remake the datadir and file self._create_ondisk_file(df, '', time()) # still empty df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger(), keep_data_fp=True) double_uuid_path = df.quarantine() self.assert_(os.path.isdir(double_uuid_path)) self.assert_('-' in os.path.basename(double_uuid_path))
def test_disk_file_app_iter_ranges_empty(self): """ This test case tests when empty value passed into app_iter_ranges When ranges passed into the method is either empty array or None, this method will yield empty string """ df = self._create_test_file('012345678911234567892123456789') it = df.app_iter_ranges([], 'application/whatever', '\r\n--someheader\r\n', 100) self.assertEqual(''.join(it), '') df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger(), keep_data_fp=True) it = df.app_iter_ranges(None, 'app/something', '\r\n--someheader\r\n', 150) self.assertEqual(''.join(it), '')
def test_hash_suffix_multi_file_one(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) for tdiff in [1, 50, 100, 500]: for suff in ['.meta', '.data', '.ts']: f = open( os.path.join( df.datadir, normalize_timestamp(int(time()) - tdiff) + suff), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hsh_path = os.listdir(whole_path_from)[0] whole_hsh_path = os.path.join(whole_path_from, hsh_path) diskfile.hash_suffix(whole_path_from, 99) # only the tombstone should be left self.assertEquals(len(os.listdir(whole_hsh_path)), 1)
def test_invalidate_hash(self): def assertFileData(file_path, data): with open(file_path, 'r') as fp: fdata = fp.read() self.assertEquals(pickle.loads(fdata), pickle.loads(data)) df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hashes_file = os.path.join(self.objects, '0', diskfile.HASH_FILE) # test that non existent file except caught self.assertEquals(diskfile.invalidate_hash(whole_path_from), None) # test that hashes get cleared check_pickle_data = pickle.dumps({data_dir: None}, diskfile.PICKLE_PROTOCOL) for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: with open(hashes_file, 'wb') as fp: pickle.dump(data_hash, fp, diskfile.PICKLE_PROTOCOL) diskfile.invalidate_hash(whole_path_from) assertFileData(hashes_file, check_pickle_data)
def test_delete_partition_with_handoff_delete_failures(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 df = diskfile.DiskFile(self.devices, 'sda', '1', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) print df.datadir f = open( os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [ node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips() ] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i in (0, 1): # force two of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() # The file should still exist self.assertTrue(os.access(part_path, os.F_OK))
def _get_disk_file(self, invalid_type=None, obj_name='o', fsize=1024, csize=8, mark_deleted=False, ts=None, iter_hook=None, mount_check=False, extra_metadata=None): '''returns a DiskFile''' df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', obj_name, FakeLogger()) data = '0' * fsize etag = md5() if ts: timestamp = ts else: timestamp = str(normalize_timestamp(time())) with df.writer() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer.fd).st_size), } metadata.update(extra_metadata or {}) writer.put(metadata) if invalid_type == 'ETag': etag = md5() etag.update('1' + '0' * (fsize - 1)) etag = etag.hexdigest() metadata['ETag'] = etag diskfile.write_metadata(writer.fd, metadata) if invalid_type == 'Content-Length': metadata['Content-Length'] = fsize - 1 diskfile.write_metadata(writer.fd, metadata) if mark_deleted: metadata = {'X-Timestamp': timestamp, 'deleted': True} df.put_metadata(metadata, tombstone=True) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', obj_name, FakeLogger(), keep_data_fp=True, disk_chunk_size=csize, iter_hook=iter_hook, mount_check=mount_check) if invalid_type == 'Zero-Byte': os.remove(df.data_file) fp = open(df.data_file, 'w') fp.close() df.unit_test_len = fsize return df
def test_disk_file_mkstemp_creates_dir(self): tmpdir = os.path.join(self.testdir, 'sda1', 'tmp') os.rmdir(tmpdir) with diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()).writer(): self.assert_(os.path.exists(tmpdir))
def object_audit(self, path, device, partition): """ Audits the given object path. :param path: a path to an object :param device: the device the path is on :param partition: the partition the path is on """ try: try: name = diskfile.read_metadata(path)['name'] except (Exception, Timeout) as exc: raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = diskfile.DiskFile(self.devices, device, partition, account, container, obj, self.logger, keep_data_fp=True) try: try: obj_size = df.get_data_file_size() except DiskFileNotExist: return except DiskFileError as e: raise AuditException(str(e)) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return for chunk in df: self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=len(chunk)) self.bytes_processed += len(chunk) self.total_bytes_processed += len(chunk) df.close() if df.quarantined_dir: self.quarantines += 1 self.logger.error( _("ERROR Object %(path)s failed audit and will be " "quarantined: ETag and file's md5 do not match"), {'path': path}) finally: df.close(verify_file=False) except AuditException as err: self.logger.increment('quarantines') self.quarantines += 1 self.logger.error( _('ERROR Object %(obj)s failed audit and will ' 'be quarantined: %(err)s'), { 'obj': path, 'err': err }) diskfile.quarantine_renamer(os.path.join(self.devices, device), path) return except (Exception, Timeout): self.logger.increment('errors') self.errors += 1 self.logger.exception(_('ERROR Trying to audit %s'), path) return self.passes += 1
def test_run_once_recover_from_timeout(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect was_get_hashes = object_replicator.get_hashes was_execute = tpool.execute self.get_hash_count = 0 try: def fake_get_hashes(*args, **kwargs): self.get_hash_count += 1 if self.get_hash_count == 3: # raise timeout on last call to get hashes raise Timeout() return 2, {'abc': 'def'} def fake_exc(tester, *args, **kwargs): if 'Error syncing partition' in args[0]: tester.i_failed = True self.i_failed = False object_replicator.http_connect = mock_http_connect(200) object_replicator.get_hashes = fake_get_hashes replicator.logger.exception = \ lambda *args, **kwargs: fake_exc(self, *args, **kwargs) # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' df = diskfile.DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) f = open( os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [ node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips() ] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue( os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) self.assertFalse(self.i_failed) finally: object_replicator.http_connect = was_connector object_replicator.get_hashes = was_get_hashes tpool.execute = was_execute