def test_connect(self): node = dict(replication_ip='1.2.3.4', replication_port=5678, device='sda1') job = dict(partition='9', policy_idx=1) self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] with mock.patch( 'swift.obj.ssync_sender.bufferedhttp.BufferedHTTPConnection' ) as mock_conn_class: mock_conn = mock_conn_class.return_value mock_resp = mock.MagicMock() mock_resp.status = 200 mock_conn.getresponse.return_value = mock_resp self.sender.connect() mock_conn_class.assert_called_once_with('1.2.3.4:5678') expectations = { 'putrequest': [ mock.call('REPLICATION', '/sda1/9'), ], 'putheader': [ mock.call('Transfer-Encoding', 'chunked'), mock.call('X-Backend-Storage-Policy-Index', 1), ], 'endheaders': [mock.call()], } for method_name, expected_calls in expectations.items(): mock_method = getattr(mock_conn, method_name) self.assertEquals(expected_calls, mock_method.mock_calls, 'connection method "%s" got %r not %r' % ( method_name, mock_method.mock_calls, expected_calls))
def test_nothing_to_sync(self): job = { 'device': self.device, 'partition': self.partition, 'policy': POLICIES.default } node = { 'replication_ip': self.rx_ip, 'replication_port': self.rx_port, 'device': self.device, 'index': 0 } sender = ssync_sender.Sender(self.daemon, node, job, ['abc']) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) result, in_sync_objs = sender() self.assertTrue(result) self.assertFalse(in_sync_objs) results = self._analyze_trace(trace) self.assertFalse(results['tx_missing']) self.assertFalse(results['rx_missing']) self.assertFalse(results['tx_updates']) self.assertFalse(results['rx_updates']) # Minimal receiver response as read by sender: # 2 <-- initial \r\n to start ssync exchange # + 23 <-- :MISSING CHECK START\r\n # + 2 <-- \r\n (minimal missing check response) # + 21 <-- :MISSING CHECK END\r\n # + 17 <-- :UPDATES START\r\n # + 15 <-- :UPDATES END\r\n # TOTAL = 80 self.assertEqual(80, trace.get('readline_bytes'))
def test_send_with_frag_index_none(self): policy = POLICIES.default tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] # create an ec fragment on the remote node ts1 = next(self.ts_iter) remote_df = self._create_ondisk_files(rx_df_mgr, 'o', policy, ts1, (3, ))[0] # create a tombstone on the local node df = self._create_ondisk_files(tx_df_mgr, 'o', policy, ts1, (3, ))[0] suffix = os.path.basename(os.path.dirname(df._datadir)) ts2 = next(self.ts_iter) df.delete(ts2) # a reconstructor revert job with only tombstones will have frag_index # explicitly set to None job = { 'frag_index': None, 'partition': self.partition, 'policy': policy, 'device': self.device, } sender = ssync_sender.Sender(self.daemon, self.rx_node, job, [suffix]) success, _ = sender() self.assertTrue(success) try: remote_df.read_metadata() except DiskFileDeleted as e: self.assertEqual(e.timestamp, ts2) else: self.fail('Successfully opened remote DiskFile')
def test_call_and_missing_check_with_obj_list_but_required(self): def yield_hashes(device, partition, policy_index, suffixes=None): if device == 'dev' and partition == '9' and suffixes == ['abc'] \ and policy_index == 0: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) job = {'device': 'dev', 'partition': '9'} self.sender = ssync_sender.Sender(self.replicator, None, job, ['abc'], ['9d41d8cd98f00b204e9800998ecf0abc']) self.sender.connection = FakeConnection() self.sender.response = FakeResponse( chunk_body=( ':MISSING_CHECK: START\r\n' '9d41d8cd98f00b204e9800998ecf0abc\r\n' ':MISSING_CHECK: END\r\n')) self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.connect = mock.MagicMock() self.sender.updates = mock.MagicMock() self.sender.disconnect = mock.MagicMock() success, candidates = self.sender() self.assertTrue(success) self.assertEqual(candidates, set())
def test_call_catches_exception_handling_exception(self): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = None # Will cause inside exception handler to fail self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' self.assertFalse(self.sender()) self.replicator.logger.exception.assert_called_once_with( 'EXCEPTION in replication.Sender')
def test_call_catches_other_exceptions(self): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' self.assertFalse(self.sender()) call = self.replicator.logger.exception.mock_calls[0] self.assertEqual(call[1], ('%s:%s/%s/%s EXCEPTION in replication.Sender', '1.2.3.4', 5678, 'sda1', '9'))
def test_call_catches_ReplicationException(self): def connect(self): raise exceptions.ReplicationException('test connect') with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual(call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), 'test connect')
def test_send_invalid_frag_index(self): policy = POLICIES.default job = { 'frag_index': 'Not a number', 'device': self.device, 'partition': self.partition, 'policy': policy } sender = ssync_sender.Sender(self.daemon, self.rx_node, job, ['abc']) success, _ = sender() self.assertFalse(success) error_log_lines = self.daemon.logger.get_lines_for_level('error') self.assertEqual(1, len(error_log_lines)) error_msg = error_log_lines[0] self.assertIn("Expected status 200; got 400", error_msg) self.assertIn("Invalid X-Backend-Ssync-Frag-Index 'Not a number'", error_msg)
def test_call_catches_MessageTimeout(self): def connect(self): exc = exceptions.MessageTimeout(1, 'test connect') # Cancels Eventlet's raising of this since we're about to do it. exc.cancel() raise exc with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual(call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '1 second: test connect')
def test_connect_send_timeout(self): self.replicator.conn_timeout = 0.01 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] def putrequest(*args, **kwargs): eventlet.sleep(0.1) with mock.patch.object( ssync_sender.bufferedhttp.BufferedHTTPConnection, 'putrequest', putrequest): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual(call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '0.01 seconds: connect send')
def test_connect_receive_timeout(self): self.replicator.node_timeout = 0.02 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): def getresponse(*args, **kwargs): eventlet.sleep(0.1) with mock.patch.object(ssync_sender.bufferedhttp, 'BufferedHTTPConnection', FakeBufferedHTTPConnection): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual(call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '0.02 seconds: connect receive')
def test_connect_bad_status(self): self.replicator.node_timeout = 0.02 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): def getresponse(*args, **kwargs): response = FakeResponse() response.status = 503 return response with mock.patch.object(ssync_sender.bufferedhttp, 'BufferedHTTPConnection', FakeBufferedHTTPConnection): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual(call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), 'Expected status 200; got 503')
def test_sync(self): policy = POLICIES.default rx_node_index = 0 # create sender side diskfiles... tx_objs = {} rx_objs = {} tx_tombstones = {} rx_tombstones = {} tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] # o1 and o2 are on tx only t1 = next(self.ts_iter) tx_objs['o1'] = self._create_ondisk_files(tx_df_mgr, 'o1', policy, t1) t2 = next(self.ts_iter) tx_objs['o2'] = self._create_ondisk_files(tx_df_mgr, 'o2', policy, t2) # o3 is on tx and older copy on rx t3a = next(self.ts_iter) rx_objs['o3'] = self._create_ondisk_files(rx_df_mgr, 'o3', policy, t3a) t3b = next(self.ts_iter) tx_objs['o3'] = self._create_ondisk_files(tx_df_mgr, 'o3', policy, t3b) # o4 in sync on rx and tx t4 = next(self.ts_iter) tx_objs['o4'] = self._create_ondisk_files(tx_df_mgr, 'o4', policy, t4) rx_objs['o4'] = self._create_ondisk_files(rx_df_mgr, 'o4', policy, t4) # o5 is a tombstone, missing on receiver t5 = next(self.ts_iter) tx_tombstones['o5'] = self._create_ondisk_files( tx_df_mgr, 'o5', policy, t5) tx_tombstones['o5'][0].delete(t5) # o6 is a tombstone, in sync on tx and rx t6 = next(self.ts_iter) tx_tombstones['o6'] = self._create_ondisk_files( tx_df_mgr, 'o6', policy, t6) tx_tombstones['o6'][0].delete(t6) rx_tombstones['o6'] = self._create_ondisk_files( rx_df_mgr, 'o6', policy, t6) rx_tombstones['o6'][0].delete(t6) # o7 is a tombstone on tx, older data on rx t7a = next(self.ts_iter) rx_objs['o7'] = self._create_ondisk_files(rx_df_mgr, 'o7', policy, t7a) t7b = next(self.ts_iter) tx_tombstones['o7'] = self._create_ondisk_files( tx_df_mgr, 'o7', policy, t7b) tx_tombstones['o7'][0].delete(t7b) suffixes = set() for diskfiles in (tx_objs.values() + tx_tombstones.values()): for df in diskfiles: suffixes.add(os.path.basename(os.path.dirname(df._datadir))) # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) # run the sync protocol... success, in_sync_objs = sender() self.assertEqual(7, len(in_sync_objs)) self.assertTrue(success) # verify protocol results = self._analyze_trace(trace) self.assertEqual(7, len(results['tx_missing'])) self.assertEqual(5, len(results['rx_missing'])) self.assertEqual(5, len(results['tx_updates'])) self.assertFalse(results['rx_updates']) sync_paths = [] for subreq in results.get('tx_updates'): if subreq.get('method') == 'PUT': self.assertTrue(subreq['path'] in ('/a/c/o1', '/a/c/o2', '/a/c/o3')) expected_body = '%s___None' % subreq['path'] self.assertEqual(expected_body, subreq['body']) elif subreq.get('method') == 'DELETE': self.assertTrue(subreq['path'] in ('/a/c/o5', '/a/c/o7')) sync_paths.append(subreq.get('path')) self.assertEqual( ['/a/c/o1', '/a/c/o2', '/a/c/o3', '/a/c/o5', '/a/c/o7'], sorted(sync_paths)) # verify on disk files... self._verify_ondisk_files(tx_objs, policy) self._verify_tombstones(tx_tombstones, policy)
def test_fragment_sync(self): # check that a sync_only type job does call reconstructor to build a # diskfile to send, and continues making progress despite an error # when building one diskfile policy = POLICIES.default rx_node_index = 0 tx_node_index = 1 # for a sync job we iterate over frag index that belongs on local node frag_index = tx_node_index # create sender side diskfiles... tx_objs = {} tx_tombstones = {} rx_objs = {} tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] # o1 only has primary t1 = next(self.ts_iter) tx_objs['o1'] = self._create_ondisk_files(tx_df_mgr, 'o1', policy, t1, (tx_node_index, )) # o2 only has primary t2 = next(self.ts_iter) tx_objs['o2'] = self._create_ondisk_files(tx_df_mgr, 'o2', policy, t2, (tx_node_index, )) # o3 only has primary t3 = next(self.ts_iter) tx_objs['o3'] = self._create_ondisk_files(tx_df_mgr, 'o3', policy, t3, (tx_node_index, )) # o4 primary fragment archives on tx, handoff in sync on rx t4 = next(self.ts_iter) tx_objs['o4'] = self._create_ondisk_files(tx_df_mgr, 'o4', policy, t4, (tx_node_index, )) rx_objs['o4'] = self._create_ondisk_files(rx_df_mgr, 'o4', policy, t4, (rx_node_index, )) # o5 is a tombstone, missing on receiver t5 = next(self.ts_iter) tx_tombstones['o5'] = self._create_ondisk_files( tx_df_mgr, 'o5', policy, t5, (tx_node_index, )) tx_tombstones['o5'][0].delete(t5) suffixes = set() for diskfiles in (tx_objs.values() + tx_tombstones.values()): for df in diskfiles: suffixes.add(os.path.basename(os.path.dirname(df._datadir))) reconstruct_fa_calls = [] def fake_reconstruct_fa(job, node, metadata): reconstruct_fa_calls.append((job, node, policy, metadata)) if len(reconstruct_fa_calls) == 2: # simulate second reconstruct failing raise DiskFileError content = '%s___%s' % (metadata['name'], rx_node_index) return RebuildingECDiskFileStream(metadata, rx_node_index, iter([content])) # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy, 'frag_index': frag_index, 'sync_diskfile_builder': fake_reconstruct_fa } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) # run the sync protocol... sender() # verify protocol results = self._analyze_trace(trace) # sender has primary for o1, o2 and o3, o4 and ts for o5 self.assertEqual(5, len(results['tx_missing'])) # receiver is missing o1, o2 and o3 and ts for o5 self.assertEqual(4, len(results['rx_missing'])) # sender can only construct 2 out of 3 missing frags self.assertEqual(3, len(results['tx_updates'])) self.assertEqual(3, len(reconstruct_fa_calls)) self.assertFalse(results['rx_updates']) actual_sync_paths = [] for subreq in results.get('tx_updates'): if subreq.get('method') == 'PUT': self.assertTrue('X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index in subreq.get('headers')) expected_body = '%s___%s' % (subreq['path'], rx_node_index) self.assertEqual(expected_body, subreq['body']) elif subreq.get('method') == 'DELETE': self.assertEqual('/a/c/o5', subreq['path']) actual_sync_paths.append(subreq.get('path')) # remove the failed df from expected synced df's expect_sync_paths = ['/a/c/o1', '/a/c/o2', '/a/c/o3', '/a/c/o5'] failed_path = reconstruct_fa_calls[1][3]['name'] expect_sync_paths.remove(failed_path) failed_obj = None for obj, diskfiles in tx_objs.items(): if diskfiles[0]._name == failed_path: failed_obj = obj # sanity check self.assertTrue(tx_objs.pop(failed_obj)) # verify on disk files... self.assertEqual(sorted(expect_sync_paths), sorted(actual_sync_paths)) self._verify_ondisk_files(tx_objs, policy, frag_index, rx_node_index) self._verify_tombstones(tx_tombstones, policy)
def test_handoff_fragment_revert(self): # test that a sync_revert type job does send the correct frag archives # to the receiver policy = POLICIES.default rx_node_index = 0 tx_node_index = 1 # for a revert job we iterate over frag index that belongs on # remote node frag_index = rx_node_index # create sender side diskfiles... tx_objs = {} rx_objs = {} tx_tombstones = {} tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] # o1 has primary and handoff fragment archives t1 = next(self.ts_iter) tx_objs['o1'] = self._create_ondisk_files( tx_df_mgr, 'o1', policy, t1, (rx_node_index, tx_node_index)) # o2 only has primary t2 = next(self.ts_iter) tx_objs['o2'] = self._create_ondisk_files(tx_df_mgr, 'o2', policy, t2, (tx_node_index, )) # o3 only has handoff t3 = next(self.ts_iter) tx_objs['o3'] = self._create_ondisk_files(tx_df_mgr, 'o3', policy, t3, (rx_node_index, )) # o4 primary and handoff fragment archives on tx, handoff in sync on rx t4 = next(self.ts_iter) tx_objs['o4'] = self._create_ondisk_files(tx_df_mgr, 'o4', policy, t4, ( tx_node_index, rx_node_index, )) rx_objs['o4'] = self._create_ondisk_files(rx_df_mgr, 'o4', policy, t4, (rx_node_index, )) # o5 is a tombstone, missing on receiver t5 = next(self.ts_iter) tx_tombstones['o5'] = self._create_ondisk_files( tx_df_mgr, 'o5', policy, t5, (tx_node_index, )) tx_tombstones['o5'][0].delete(t5) suffixes = set() for diskfiles in (tx_objs.values() + tx_tombstones.values()): for df in diskfiles: suffixes.add(os.path.basename(os.path.dirname(df._datadir))) # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy, 'frag_index': frag_index } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) # run the sync protocol... sender() # verify protocol results = self._analyze_trace(trace) # sender has handoff frags for o1, o3 and o4 and ts for o5 self.assertEqual(4, len(results['tx_missing'])) # receiver is missing frags for o1, o3 and ts for o5 self.assertEqual(3, len(results['rx_missing'])) self.assertEqual(3, len(results['tx_updates'])) self.assertFalse(results['rx_updates']) sync_paths = [] for subreq in results.get('tx_updates'): if subreq.get('method') == 'PUT': self.assertTrue('X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index in subreq.get('headers')) expected_body = '%s___%s' % (subreq['path'], rx_node_index) self.assertEqual(expected_body, subreq['body']) elif subreq.get('method') == 'DELETE': self.assertEqual('/a/c/o5', subreq['path']) sync_paths.append(subreq.get('path')) self.assertEqual(['/a/c/o1', '/a/c/o3', '/a/c/o5'], sorted(sync_paths)) # verify on disk files... self._verify_ondisk_files(tx_objs, policy, frag_index, rx_node_index) self._verify_tombstones(tx_tombstones, policy)
def ssync(self, node, job, suffixes): return ssync_sender.Sender(self, node, job, suffixes)()
def test_meta_file_sync(self): policy = POLICIES.default rx_node_index = 0 # create diskfiles... tx_objs = {} rx_objs = {} tx_tombstones = {} rx_tombstones = {} tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] expected_subreqs = defaultdict(list) # o1 on tx only with meta file t1 = next(self.ts_iter) tx_objs['o1'] = self._create_ondisk_files(tx_df_mgr, 'o1', policy, t1) t1_meta = next(self.ts_iter) metadata = { 'X-Timestamp': t1_meta.internal, 'X-Object-Meta-Test': 'o1', 'X-Object-Sysmeta-Test': 'sys_o1' } tx_objs['o1'][0].write_metadata(metadata) expected_subreqs['PUT'].append('o1') expected_subreqs['POST'].append('o1') # o2 on tx with meta, on rx without meta t2 = next(self.ts_iter) tx_objs['o2'] = self._create_ondisk_files(tx_df_mgr, 'o2', policy, t2) t2_meta = next(self.ts_iter) metadata = { 'X-Timestamp': t2_meta.internal, 'X-Object-Meta-Test': 'o2', 'X-Object-Sysmeta-Test': 'sys_o2' } tx_objs['o2'][0].write_metadata(metadata) rx_objs['o2'] = self._create_ondisk_files(rx_df_mgr, 'o2', policy, t2) expected_subreqs['POST'].append('o2') # o3 is on tx with meta, rx has newer data but no meta t3a = next(self.ts_iter) tx_objs['o3'] = self._create_ondisk_files(tx_df_mgr, 'o3', policy, t3a) t3b = next(self.ts_iter) rx_objs['o3'] = self._create_ondisk_files(rx_df_mgr, 'o3', policy, t3b) t3_meta = next(self.ts_iter) metadata = { 'X-Timestamp': t3_meta.internal, 'X-Object-Meta-Test': 'o3', 'X-Object-Sysmeta-Test': 'sys_o3' } tx_objs['o3'][0].write_metadata(metadata) expected_subreqs['POST'].append('o3') # o4 is on tx with meta, rx has older data and up to date meta t4a = next(self.ts_iter) rx_objs['o4'] = self._create_ondisk_files(rx_df_mgr, 'o4', policy, t4a) t4b = next(self.ts_iter) tx_objs['o4'] = self._create_ondisk_files(tx_df_mgr, 'o4', policy, t4b) t4_meta = next(self.ts_iter) metadata = { 'X-Timestamp': t4_meta.internal, 'X-Object-Meta-Test': 'o4', 'X-Object-Sysmeta-Test': 'sys_o4' } tx_objs['o4'][0].write_metadata(metadata) rx_objs['o4'][0].write_metadata(metadata) expected_subreqs['PUT'].append('o4') # o5 is on tx with meta, rx is in sync with data and meta t5 = next(self.ts_iter) rx_objs['o5'] = self._create_ondisk_files(rx_df_mgr, 'o5', policy, t5) tx_objs['o5'] = self._create_ondisk_files(tx_df_mgr, 'o5', policy, t5) t5_meta = next(self.ts_iter) metadata = { 'X-Timestamp': t5_meta.internal, 'X-Object-Meta-Test': 'o5', 'X-Object-Sysmeta-Test': 'sys_o5' } tx_objs['o5'][0].write_metadata(metadata) rx_objs['o5'][0].write_metadata(metadata) # o6 is tombstone on tx, rx has older data and meta t6 = next(self.ts_iter) tx_tombstones['o6'] = self._create_ondisk_files( tx_df_mgr, 'o6', policy, t6) rx_tombstones['o6'] = self._create_ondisk_files( rx_df_mgr, 'o6', policy, t6) metadata = { 'X-Timestamp': next(self.ts_iter).internal, 'X-Object-Meta-Test': 'o6', 'X-Object-Sysmeta-Test': 'sys_o6' } rx_tombstones['o6'][0].write_metadata(metadata) tx_tombstones['o6'][0].delete(next(self.ts_iter)) expected_subreqs['DELETE'].append('o6') # o7 is tombstone on rx, tx has older data and meta, # no subreqs expected... t7 = next(self.ts_iter) tx_objs['o7'] = self._create_ondisk_files(tx_df_mgr, 'o7', policy, t7) rx_tombstones['o7'] = self._create_ondisk_files( rx_df_mgr, 'o7', policy, t7) metadata = { 'X-Timestamp': next(self.ts_iter).internal, 'X-Object-Meta-Test': 'o7', 'X-Object-Sysmeta-Test': 'sys_o7' } tx_objs['o7'][0].write_metadata(metadata) rx_tombstones['o7'][0].delete(next(self.ts_iter)) suffixes = set() for diskfiles in (tx_objs.values() + tx_tombstones.values()): for df in diskfiles: suffixes.add(os.path.basename(os.path.dirname(df._datadir))) # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) # run the sync protocol... success, in_sync_objs = sender() self.assertEqual(7, len(in_sync_objs)) self.assertTrue(success) # verify protocol results = self._analyze_trace(trace) self.assertEqual(7, len(results['tx_missing'])) self.assertEqual(5, len(results['rx_missing'])) for subreq in results.get('tx_updates'): obj = subreq['path'].split('/')[3] method = subreq['method'] self.assertTrue( obj in expected_subreqs[method], 'Unexpected %s subreq for object %s, expected %s' % (method, obj, expected_subreqs[method])) expected_subreqs[method].remove(obj) if method == 'PUT': expected_body = '%s___None' % subreq['path'] self.assertEqual(expected_body, subreq['body']) # verify all expected subreqs consumed for _method, expected in expected_subreqs.items(): self.assertFalse(expected) self.assertFalse(results['rx_updates']) # verify on disk files... del tx_objs['o7'] # o7 not expected to be sync'd self._verify_ondisk_files(tx_objs, policy) self._verify_tombstones(tx_tombstones, policy) for oname, rx_obj in rx_objs.items(): df = rx_obj[0].open() metadata = df.get_metadata() self.assertEqual(metadata['X-Object-Meta-Test'], oname) self.assertEqual(metadata['X-Object-Sysmeta-Test'], 'sys_' + oname)
def test_meta_file_not_synced_to_legacy_receiver(self): # verify that the sender does sync a data file to a legacy receiver, # but does not PUT meta file content to a legacy receiver policy = POLICIES.default rx_node_index = 0 # create diskfiles... tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] # rx has data at t1 but no meta # object is on tx with data at t2, meta at t3, t1 = next(self.ts_iter) self._create_ondisk_files(rx_df_mgr, 'o1', policy, t1) t2 = next(self.ts_iter) tx_obj = self._create_ondisk_files(tx_df_mgr, 'o1', policy, t2)[0] t3 = next(self.ts_iter) metadata = { 'X-Timestamp': t3.internal, 'X-Object-Meta-Test': 'o3', 'X-Object-Sysmeta-Test': 'sys_o3' } tx_obj.write_metadata(metadata) suffixes = [os.path.basename(os.path.dirname(tx_obj._datadir))] # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) def _legacy_check_missing(self, line): # reproduces behavior of 'legacy' ssync receiver missing_checks() parts = line.split() object_hash = urllib.parse.unquote(parts[0]) timestamp = urllib.parse.unquote(parts[1]) want = False try: df = self.diskfile_mgr.get_diskfile_from_hash( self.device, self.partition, object_hash, self.policy, frag_index=self.frag_index) except DiskFileNotExist: want = True else: try: df.open() except DiskFileDeleted as err: want = err.timestamp < timestamp except DiskFileError: want = True else: want = df.timestamp < timestamp if want: return urllib.parse.quote(object_hash) return None # run the sync protocol... func = 'swift.obj.ssync_receiver.Receiver._check_missing' with mock.patch(func, _legacy_check_missing): success, in_sync_objs = sender() self.assertEqual(1, len(in_sync_objs)) self.assertTrue(success) # verify protocol, expecting only a PUT to legacy receiver results = self._analyze_trace(trace) self.assertEqual(1, len(results['tx_missing'])) self.assertEqual(1, len(results['rx_missing'])) self.assertEqual(1, len(results['tx_updates'])) self.assertEqual('PUT', results['tx_updates'][0]['method']) self.assertFalse(results['rx_updates']) # verify on disk files... rx_obj = self._open_rx_diskfile('o1', policy) tx_obj = self._open_tx_diskfile('o1', policy) # with legacy behavior rx_obj data and meta timestamps are equal self.assertEqual(t2, rx_obj.data_timestamp) self.assertEqual(t2, rx_obj.timestamp) # with legacy behavior rx_obj data timestamp should equal tx_obj self.assertEqual(rx_obj.data_timestamp, tx_obj.data_timestamp) # tx meta file should not have been sync'd to rx data file self.assertNotIn('X-Object-Meta-Test', rx_obj.get_metadata())
def setUp(self): self.tmpdir = tempfile.mkdtemp() self.testdir = os.path.join(self.tmpdir, 'tmp_test_ssync_sender') self.replicator = FakeReplicator(self.testdir) self.sender = ssync_sender.Sender(self.replicator, None, None, None)
def ssync(self, node, job, suffixes, remote_check_objs=None): return ssync_sender.Sender(self, node, job, suffixes, remote_check_objs)()
def test_content_type_sync(self): policy = POLICIES.default rx_node_index = 0 # create diskfiles... tx_objs = {} rx_objs = {} tx_df_mgr = self.daemon._diskfile_router[policy] rx_df_mgr = self.rx_controller._diskfile_router[policy] expected_subreqs = defaultdict(list) # o1 on tx only with two meta files name = 'o1' t1 = self.ts_iter.next() tx_objs[name] = self._create_ondisk_files(tx_df_mgr, name, policy, t1) t1_type = self.ts_iter.next() metadata_1 = { 'X-Timestamp': t1_type.internal, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t1_type.internal } tx_objs[name][0].write_metadata(metadata_1) t1_meta = self.ts_iter.next() metadata_2 = { 'X-Timestamp': t1_meta.internal, 'X-Object-Meta-Test': name } tx_objs[name][0].write_metadata(metadata_2) expected_subreqs['PUT'].append(name) expected_subreqs['POST'].append(name) # o2 on tx with two meta files, rx has .data and newest .meta but is # missing latest content-type name = 'o2' t2 = self.ts_iter.next() tx_objs[name] = self._create_ondisk_files(tx_df_mgr, name, policy, t2) t2_type = self.ts_iter.next() metadata_1 = { 'X-Timestamp': t2_type.internal, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t2_type.internal } tx_objs[name][0].write_metadata(metadata_1) t2_meta = self.ts_iter.next() metadata_2 = { 'X-Timestamp': t2_meta.internal, 'X-Object-Meta-Test': name } tx_objs[name][0].write_metadata(metadata_2) rx_objs[name] = self._create_ondisk_files(rx_df_mgr, name, policy, t2) rx_objs[name][0].write_metadata(metadata_2) expected_subreqs['POST'].append(name) # o3 on tx with two meta files, rx has .data and one .meta but does # have latest content-type so nothing to sync name = 'o3' t3 = self.ts_iter.next() tx_objs[name] = self._create_ondisk_files(tx_df_mgr, name, policy, t3) t3_type = self.ts_iter.next() metadata_1 = { 'X-Timestamp': t3_type.internal, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t3_type.internal } tx_objs[name][0].write_metadata(metadata_1) t3_meta = self.ts_iter.next() metadata_2 = { 'X-Timestamp': t3_meta.internal, 'X-Object-Meta-Test': name } tx_objs[name][0].write_metadata(metadata_2) rx_objs[name] = self._create_ondisk_files(rx_df_mgr, name, policy, t3) metadata_2b = { 'X-Timestamp': t3_meta.internal, 'X-Object-Meta-Test': name, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t3_type.internal } rx_objs[name][0].write_metadata(metadata_2b) # o4 on tx with one meta file having latest content-type, rx has # .data and two .meta having latest content-type so nothing to sync # i.e. o4 is the reverse of o3 scenario name = 'o4' t4 = self.ts_iter.next() tx_objs[name] = self._create_ondisk_files(tx_df_mgr, name, policy, t4) t4_type = self.ts_iter.next() t4_meta = self.ts_iter.next() metadata_2b = { 'X-Timestamp': t4_meta.internal, 'X-Object-Meta-Test': name, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t4_type.internal } tx_objs[name][0].write_metadata(metadata_2b) rx_objs[name] = self._create_ondisk_files(rx_df_mgr, name, policy, t4) metadata_1 = { 'X-Timestamp': t4_type.internal, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t4_type.internal } rx_objs[name][0].write_metadata(metadata_1) metadata_2 = { 'X-Timestamp': t4_meta.internal, 'X-Object-Meta-Test': name } rx_objs[name][0].write_metadata(metadata_2) # o5 on tx with one meta file having latest content-type, rx has # .data and no .meta name = 'o5' t5 = self.ts_iter.next() tx_objs[name] = self._create_ondisk_files(tx_df_mgr, name, policy, t5) t5_type = self.ts_iter.next() t5_meta = self.ts_iter.next() metadata = { 'X-Timestamp': t5_meta.internal, 'X-Object-Meta-Test': name, 'Content-Type': 'text/test', 'Content-Type-Timestamp': t5_type.internal } tx_objs[name][0].write_metadata(metadata) rx_objs[name] = self._create_ondisk_files(rx_df_mgr, name, policy, t5) expected_subreqs['POST'].append(name) suffixes = set() for diskfiles in tx_objs.values(): for df in diskfiles: suffixes.add(os.path.basename(os.path.dirname(df._datadir))) # create ssync sender instance... job = { 'device': self.device, 'partition': self.partition, 'policy': policy } node = dict(self.rx_node) node.update({'index': rx_node_index}) sender = ssync_sender.Sender(self.daemon, node, job, suffixes) # wrap connection from tx to rx to capture ssync messages... sender.connect, trace = self.make_connect_wrapper(sender) # run the sync protocol... success, in_sync_objs = sender() self.assertEqual(5, len(in_sync_objs), trace['messages']) self.assertTrue(success) # verify protocol results = self._analyze_trace(trace) self.assertEqual(5, len(results['tx_missing'])) self.assertEqual(3, len(results['rx_missing'])) for subreq in results.get('tx_updates'): obj = subreq['path'].split('/')[3] method = subreq['method'] self.assertTrue( obj in expected_subreqs[method], 'Unexpected %s subreq for object %s, expected %s' % (method, obj, expected_subreqs[method])) expected_subreqs[method].remove(obj) if method == 'PUT': expected_body = '%s___None' % subreq['path'] self.assertEqual(expected_body, subreq['body']) # verify all expected subreqs consumed for _method, expected in expected_subreqs.items(): self.assertFalse( expected, 'Expected subreqs not seen for %s for objects %s' % (_method, expected)) self.assertFalse(results['rx_updates']) # verify on disk files... self._verify_ondisk_files(tx_objs, policy) for oname, rx_obj in rx_objs.items(): df = rx_obj[0].open() metadata = df.get_metadata() self.assertEqual(metadata['X-Object-Meta-Test'], oname) self.assertEqual(metadata['Content-Type'], 'text/test') # verify that tx and rx both generate the same suffix hashes... tx_hashes = tx_df_mgr.get_hashes(self.device, self.partition, suffixes, policy) rx_hashes = rx_df_mgr.get_hashes(self.device, self.partition, suffixes, policy) self.assertEqual(tx_hashes, rx_hashes)