def test_add_to_transfer_queue(gcbl): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block gcbl.return_value = None s._add_to_transfer_queue(src_ase, dst_ase) assert gcbl.call_count == 1 assert s._transfer_queue.qsize() == 1 assert s._synccopy_start_time is not None src_ase.mode = azmodels.StorageModes.Page s._add_to_transfer_queue(src_ase, dst_ase) assert gcbl.call_count == 1 assert s._transfer_queue.qsize() == 2 assert s._synccopy_start_time is not None
def test_finalize_upload(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.Block ase.replica_targets = [ase] ase.md5 = 'md5' sd = mock.MagicMock() sd.dst_entity = ase sd.src_entity = ase s._finalize_block_blob = mock.MagicMock() s._finalize_upload(sd) assert s._finalize_block_blob.call_count == 1 ase.md5 = None s._finalize_upload(sd) assert s._finalize_block_blob.call_count == 2 sd.requires_put_block_list = False sd.remote_is_page_blob = True s._finalize_nonblock_blob = mock.MagicMock() s._finalize_upload(sd) assert s._finalize_nonblock_blob.call_count == 1 sd.remote_is_page_blob = False sd.remote_is_append_blob = False sd.remote_is_file = True s._finalize_azure_file = mock.MagicMock() s._finalize_upload(sd) assert s._finalize_azure_file.call_count == 1
def test_bind_sources_to_destination(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False s._spec.options.strip_components = 0 s._spec.options.delete_extraneous_destination = True src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.container = 'srccont' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.name = 'srcase' dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.container = 'dstcont' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block dst_ase.size = 10 dst_ase.name = 'dstase' dst_ase.from_local = False dst_ase.replica_targets = None dst2_ase = mock.MagicMock() dst2_ase._client.primary_endpoint = 'ep2a' dst2_ase.container = 'dstcont2' dst2_ase.name = 'dstase2' src = mock.MagicMock() src.files.return_value = [src_ase] s._spec.sources = [src] s._generate_destination_for_source = mock.MagicMock() i = 0 for a, b in s._bind_sources_to_destination(): i += 1 assert i == 0 s._generate_destination_for_source.return_value = [dst_ase, dst2_ase] a, b = next(s._bind_sources_to_destination()) assert a == src_ase assert b == dst_ase assert len(b.replica_targets) == 1 assert b.replica_targets[0] == dst2_ase dst_ase.replica_targets = [dst2_ase] s._generate_destination_for_source.return_value = [dst_ase, dst2_ase] with pytest.raises(RuntimeError): a, b = next(s._bind_sources_to_destination()) dst_ase.replica_targets = None s._spec.options.delete_extraneous_destination = False src.files.return_value = [src_ase, src_ase] s._generate_destination_for_source.return_value = [dst_ase] with pytest.raises(RuntimeError): for a, b in s._bind_sources_to_destination(): pass
def test_put_data(pp, pfr, pb, cb, ab): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False offsets = mock.MagicMock() offsets.chunk_num = 0 offsets.num_bytes = 1 offsets.range_start = 10 src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.is_encrypted = False dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Append dst_ase.size = 10 dst_ase.is_encrypted = False sd = mock.MagicMock() sd.src_entity = src_ase sd.dst_entity = dst_ase sd.complete_offset_upload = mock.MagicMock() s._put_data(sd, dst_ase, offsets, b'\0') assert ab.call_count == 1 dst_ase.mode = azmodels.StorageModes.Block sd.is_one_shot_block_blob = True sd.must_compute_md5 = True sd.src_entity.md5 = '' s._put_data(sd, dst_ase, offsets, b'\0') assert cb.call_count == 1 sd.src_entity.md5 = b'md5' s._put_data(sd, dst_ase, offsets, b'\0') assert cb.call_count == 2 sd.must_compute_md5 = False s._put_data(sd, dst_ase, offsets, b'\0') assert cb.call_count == 3 sd.is_one_shot_block_blob = False s._put_data(sd, dst_ase, offsets, b'\0') assert pb.call_count == 1 dst_ase.mode = azmodels.StorageModes.File s._put_data(sd, dst_ase, offsets, b'\0') assert pfr.call_count == 1 dst_ase.mode = azmodels.StorageModes.Page s._put_data(sd, dst_ase, offsets, b'\0' * 512) assert pp.call_count == 0 s._put_data(sd, dst_ase, offsets, b'1') assert pp.call_count == 1
def test_prepare_upload(page_cb, cf, capd, cs, append_cb, cc): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.Append ase.append_create = True s._prepare_upload(ase) assert cc.call_count == 1 assert append_cb.call_count == 1 ase.mode = azmodels.StorageModes.Block ase.append_create = False s._prepare_upload(ase) assert cc.call_count == 2 ase.mode = azmodels.StorageModes.File s._prepare_upload(ase) assert cs.call_count == 1 assert capd.call_count == 1 assert cf.call_count == 1 ase.mode = azmodels.StorageModes.Page s._prepare_upload(ase) assert cc.call_count == 3 assert page_cb.call_count == 1
def test_update_progress_bar(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) with mock.patch( 'blobxfer.operations.progress.update_progress_bar') as patched_upb: s._update_progress_bar() assert patched_upb.call_count == 1
def test_run(srm, gbr, gfr): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._general_options.concurrency.transfer_threads = 1 s._general_options.resume_file = 'resume' s._spec.options.chunk_size_bytes = 0 src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.container = 'srccont' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.name = 'srcase' dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.container = 'dstcont' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block dst_ase.size = 10 dst_ase.name = 'dstase' dst_ase.from_local = False dst_ase.replica_targets = None s._bind_sources_to_destination = mock.MagicMock() s._bind_sources_to_destination.return_value = [(src_ase, dst_ase)] s._prepare_upload = mock.MagicMock() s._put_data = mock.MagicMock() s._finalize_upload = mock.MagicMock() # normal execution s._synccopy_start_time = (util.datetime_now() - datetime.timedelta(seconds=1)) s._run() assert s._prepare_upload.call_count == 1 assert s._put_data.call_count == 1 # dry run s._general_options.dry_run = True s._run() assert s._prepare_upload.call_count == 1 assert s._put_data.call_count == 1 s._general_options.dry_run = False # replica targets with mismatch s._synccopy_start_time = None dst_ase.replica_targets = [dst_ase] with pytest.raises(RuntimeError): s._run() # exception during worker thread dst_ase.replica_targets = None with pytest.raises(RuntimeError): s._process_synccopy_descriptor = mock.MagicMock() s._process_synccopy_descriptor.side_effect = RuntimeError() s._run()
def test_generate_destination_for_source(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.dest_mode = azmodels.StorageModes.Block s._spec.options.rename = False s._check_for_existing_remote = mock.MagicMock() s._check_for_existing_remote.return_value = None src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.name = 'srcase' sa = mock.MagicMock() sa.block_blob_client.primary_endpoint = 'ep' sa.name = 'name' sa.endpoint = 'ep' s._get_destination_paths = mock.MagicMock() s._get_destination_paths.return_value = [ (sa, 'cont', 'dstname', 'dpath'), ] s._check_copy_conditions = mock.MagicMock() s._check_copy_conditions.return_value = ops.SynccopyAction.Skip s._general_options.dry_run = True with pytest.raises(StopIteration): next(s._generate_destination_for_source(src_ase)) assert len(s._delete_exclude) == 1 s._general_options.dry_run = False s._delete_exclude = set() s._check_copy_conditions.return_value = ops.SynccopyAction.Copy ase = next(s._generate_destination_for_source(src_ase)) assert ase is not None assert len(s._delete_exclude) == 0 assert ase.size == src_ase.size assert ase.mode == s._spec.options.dest_mode assert pathlib.Path(ase.name) == pathlib.Path('dstname', src_ase.name) s._get_destination_paths.return_value = [ (sa, 'cont', 'name', 'dpath'), ] s._spec.options.rename = True ase = next(s._generate_destination_for_source(src_ase)) assert ase.name == 'name' s._get_destination_paths.return_value = [ (sa, 'cont', '', 'dpath'), ] s._spec.options.rename = True with pytest.raises(RuntimeError): next(s._generate_destination_for_source(src_ase))
def test_finalize_nonblock_blob(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._set_blob_md5 = mock.MagicMock() s._set_blob_metadata = mock.MagicMock() s._finalize_nonblock_blob(mock.MagicMock(), {'a': 0}, 'digest') assert s._set_blob_md5.call_count == 1 assert s._set_blob_metadata.call_count == 1
def test_get_destination_paths(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) paths = mock.MagicMock() paths.paths = [pathlib.Path('a/b')] s._spec.destinations = [paths] sa, cont, dir, dpath = next(s._get_destination_paths()) assert cont == 'a' assert dir == 'b' assert dpath == pathlib.Path('a/b')
def test_finalize_nonblock_blob(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._set_blob_properties = mock.MagicMock() s._set_blob_metadata = mock.MagicMock() s._finalize_nonblock_blob(mock.MagicMock(), {'a': 0}, 'digest') assert s._set_blob_properties.call_count == 1 assert s._set_blob_metadata.call_count == 1
def test_initialize_transfer_threads(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.concurrency.transfer_threads = 1 try: s._initialize_transfer_threads() assert len(s._transfer_threads) == 1 finally: s._wait_for_transfer_threads(True) for thr in s._transfer_threads: assert not thr.is_alive()
def test_delete_extraneous_files(db, lab, df, laf): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) # test no delete s._spec.options.delete_extraneous_destination = False s._get_destination_paths = mock.MagicMock() s._delete_extraneous_files() assert s._get_destination_paths.call_count == 0 # test file delete s._spec.options.delete_extraneous_destination = True s._spec.options.dest_mode = azmodels.StorageModes.File sa1 = mock.MagicMock() sa1.name = 'name' sa1.endpoint = 'ep' sa1.file_client.primary_endpoint = 'ep' s._get_destination_paths = mock.MagicMock() s._get_destination_paths.return_value = [ (sa1, 'cont', None, None), (sa1, 'cont', None, None), ] laf.return_value = ['filename'] s._delete_extraneous_files() assert laf.call_count == 1 assert df.call_count == 1 # test blob delete s._spec.options.delete_extraneous_destination = True s._spec.options.dest_mode = azmodels.StorageModes.Block sa1 = mock.MagicMock() sa1.name = 'name' sa1.endpoint = 'ep' sa1.block_blob_client.primary_endpoint = 'ep' s._get_destination_paths = mock.MagicMock() s._get_destination_paths.return_value = [ (sa1, 'cont', None, None), ] blob = mock.MagicMock() blob.name = 'blobname' lab.return_value = [blob] s._delete_extraneous_files() assert lab.call_count == 1 assert db.call_count == 1
def test_worker_thread_transfer(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._transfer_queue.put(mock.MagicMock()) s._transfer_queue.put(mock.MagicMock()) s._process_synccopy_descriptor = mock.MagicMock() s._process_synccopy_descriptor.side_effect = [None, Exception()] with mock.patch('blobxfer.operations.synccopy.SyncCopy.termination_check', new_callable=mock.PropertyMock) as patched_tc: patched_tc.side_effect = [False, False, True] s._worker_thread_transfer() assert s._process_synccopy_descriptor.call_count == 2 assert len(s._exceptions) == 1
def test_process_data(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._put_data = mock.MagicMock() s._complete_offset_upload = mock.MagicMock() offsets = mock.MagicMock() offsets.num_bytes = 1 offsets.chunk_num = 0 sd = mock.MagicMock() s._process_data(sd, mock.MagicMock(), offsets, mock.MagicMock()) assert s._put_data.call_count == 1 assert s._synccopy_bytes_sofar == 1 assert sd.complete_offset_upload.call_count == 1
def test_get_destination_paths(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False s._spec.options.strip_components = 0 paths = mock.MagicMock() paths.paths = [pathlib.Path('a/b')] s._spec.destinations = [paths] sa, cont, dir, dpath = next(s._get_destination_paths()) assert cont == 'a' assert dir == 'b' assert dpath == pathlib.Path('a/b')
def test_global_dest_mode_is_file(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._spec.options.dest_mode = azmodels.StorageModes.File assert s._global_dest_mode_is_file() s._spec.options.dest_mode = azmodels.StorageModes.Auto s._spec.options.mode = azmodels.StorageModes.File assert s._global_dest_mode_is_file() s._spec.options.mode = azmodels.StorageModes.Page assert not s._global_dest_mode_is_file()
def test_set_blob_metadata(sbm): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.Block ase.is_encrypted = False ase.replica_targets = [ase] sd = mock.MagicMock() sd.dst_entity = ase s._set_blob_metadata(sd, mock.MagicMock()) assert sbm.call_count == 2
def test_finalize_azure_file(sfmeta, sfmd5): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.File ase.replica_targets = [ase] sd = mock.MagicMock() sd.dst_entity = ase s._finalize_azure_file(sd, {'a': 0}, 'md5') assert sfmd5.call_count == 2 assert sfmeta.call_count == 2
def test_finalize_block_blob(pbl): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.Block ase.is_encrypted = False ase.replica_targets = [ase] sd = mock.MagicMock() sd.dst_entity = ase sd.last_block_num = 1 s._finalize_block_blob(sd, mock.MagicMock(), mock.MagicMock()) assert pbl.call_count == 2
def test_set_blob_properties(sbp): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.Block ase.is_encrypted = False ase.replica_targets = [ase] sd = mock.MagicMock() sd.dst_entity = ase s._set_blob_properties(sd, mock.MagicMock()) assert sbp.call_count == 2
def test_check_for_existing_remote(gbp, gfp): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False s._spec.options.strip_components = 0 sa = mock.MagicMock() sa.name = 'name' sa.endpoint = 'ep' s._spec.options.dest_mode = azmodels.StorageModes.File gfp.return_value = None src_mode = azmodels.StorageModes.File assert s._check_for_existing_remote(sa, 'cont', 'name', src_mode) is None with mock.patch( 'blobxfer.models.crypto.EncryptionMetadata.' 'encryption_metadata_exists', return_value=False): gfp.return_value = mock.MagicMock() assert s._check_for_existing_remote(sa, 'cont', 'name', src_mode) is not None with mock.patch( 'blobxfer.models.crypto.EncryptionMetadata.' 'encryption_metadata_exists', return_value=True): with mock.patch( 'blobxfer.models.crypto.EncryptionMetadata.convert_from_json'): gfp.return_value = mock.MagicMock() assert s._check_for_existing_remote(sa, 'cont', 'name', src_mode) is not None s._spec.options.dest_mode = azmodels.StorageModes.Block gbp.return_value = None src_mode = azmodels.StorageModes.Block assert s._check_for_existing_remote(sa, 'cont', 'name', src_mode) is None with mock.patch( 'blobxfer.models.crypto.EncryptionMetadata.' 'encryption_metadata_exists', return_value=False): gbp.return_value = mock.MagicMock() assert s._check_for_existing_remote(sa, 'cont', 'name', src_mode) is not None
def test_start(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._wait_for_transfer_threads = mock.MagicMock() s._resume = mock.MagicMock() s._run = mock.MagicMock() # test keyboard interrupt s._run.side_effect = KeyboardInterrupt() s.start() assert s._run.call_count == 1 assert s._wait_for_transfer_threads.call_count == 1 assert s._resume.close.call_count == 1 # test other exception s._run.side_effect = RuntimeError() s.start()
def test_finalize_azure_file(sfmeta, sfp): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.server_side_copy = False ase = mock.MagicMock() ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.size = 10 ase.mode = azmodels.StorageModes.File ase.replica_targets = [ase] sd = mock.MagicMock() sd.dst_entity = ase s._finalize_azure_file(sd, {'a': 0}, 'md5') assert sfp.call_count == 2 assert sfmeta.call_count == 2
def test_generate_destination_for_source(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._spec.options.dest_mode = azmodels.StorageModes.Block s._spec.options.rename = False s._check_for_existing_remote = mock.MagicMock() s._check_for_existing_remote.return_value = None src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.name = 'srcase' sa = mock.MagicMock() sa.name = 'name' sa.endpoint = 'ep' s._get_destination_paths = mock.MagicMock() s._get_destination_paths.return_value = [ (sa, 'cont', 'dstname', 'dpath'), ] s._check_copy_conditions = mock.MagicMock() s._check_copy_conditions.return_value = ops.SynccopyAction.Copy ase = next(s._generate_destination_for_source(src_ase)) assert ase is not None assert ase.size == src_ase.size assert ase.mode == s._spec.options.dest_mode assert pathlib.Path(ase.name) == pathlib.Path('dstname', src_ase.name) s._get_destination_paths.return_value = [ (sa, 'cont', 'name', 'dpath'), ] s._spec.options.rename = True ase = next(s._generate_destination_for_source(src_ase)) assert ase.name == 'name'
def test_translate_src_mode_to_dst_mode(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._spec.options.dest_mode = azmodels.StorageModes.Auto src_mode = azmodels.StorageModes.File assert s._translate_src_mode_to_dst_mode( src_mode) == azmodels.StorageModes.File src_mode = azmodels.StorageModes.Append assert s._translate_src_mode_to_dst_mode( src_mode) == azmodels.StorageModes.Append s._spec.options.dest_mode = azmodels.StorageModes.Page assert s._translate_src_mode_to_dst_mode( src_mode) == azmodels.StorageModes.Page s._spec.options.dest_mode = azmodels.StorageModes.File assert s._translate_src_mode_to_dst_mode( src_mode) == azmodels.StorageModes.File
def test_start(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._spec.options.delete_only = False s._spec.options.server_side_copy = False s._spec.options.strip_components = 0 s._wait_for_transfer_threads = mock.MagicMock() s._resume = mock.MagicMock() s._run = mock.MagicMock() # test keyboard interrupt s._run.side_effect = KeyboardInterrupt() with pytest.raises(KeyboardInterrupt): s.start() assert s._run.call_count == 1 assert s._wait_for_transfer_threads.call_count == 1 assert s._resume.close.call_count == 1 # test other exception s._run.side_effect = RuntimeError() with pytest.raises(RuntimeError): s.start() assert s._run.call_count == 2 assert s._wait_for_transfer_threads.call_count == 2 assert s._resume.close.call_count == 2 with pytest.raises(RuntimeError): s._wait_for_transfer_threads = mock.MagicMock( side_effect=RuntimeError('oops')) s.start() assert s._run.call_count == 3 assert s._wait_for_transfer_threads.call_count == 1 assert s._resume.close.call_count == 3
def test_check_copy_conditions(gmfm): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block dst_ase.size = 10 dst_ase.from_local = False assert s._check_copy_conditions(src_ase, None) == ops.SynccopyAction.Copy s._spec.options.overwrite = False assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Skip s._spec.options.overwrite = True s._spec.skip_on.md5_match = True gmfm.return_value = 'md5' assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Skip gmfm.side_effect = ['md50', 'md51'] assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Copy gmfm.return_value = None gmfm.side_effect = None s._spec.skip_on.md5_match = False s._spec.skip_on.filesize_match = False s._spec.skip_on.lmt_ge = False assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Copy # size mismatch s._spec.skip_on.filesize_match = True src_ase.size = 1 assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Copy # size match s._spec.skip_on.filesize_match = True src_ase.size = dst_ase.size assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Skip # lmt match s._spec.skip_on.filesize_match = False s._spec.skip_on.lmt_ge = True src_ase.lmt = 0 dst_ase.lmt = 0 assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Skip # lmt mismatch s._spec.skip_on.lmt_ge = True src_ase.lmt = 1 assert s._check_copy_conditions( src_ase, dst_ase) == ops.SynccopyAction.Copy
def test_termination_check(): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) assert not s.termination_check
def test_process_synccopy_descriptor(gbr, gfr): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.is_encrypted = False dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block dst_ase.size = 10 dst_ase.is_encrypted = False sd = mock.MagicMock() sd.src_entity = src_ase sd.dst_entity = dst_ase sd.complete_offset_upload = mock.MagicMock() sd.next_offsets.return_value = (None, 1) sd.is_one_shot_block_blob = False sd.all_operations_completed = True s._finalize_upload = mock.MagicMock() s._transfer_set.add( ops.SyncCopy.create_unique_transfer_operation_id(src_ase, dst_ase)) # test resume and completed s._process_synccopy_descriptor(sd) assert s._synccopy_bytes_sofar == 1 assert s._finalize_upload.call_count == 1 assert len(s._transfer_set) == 0 assert s._synccopy_sofar == 1 # test nothing s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False sd.all_operations_completed = False sd.next_offsets.return_value = (None, None) s._process_synccopy_descriptor(sd) assert s._transfer_queue.qsize() == 1 # test normal block blob s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._prepare_upload = mock.MagicMock() s._process_data = mock.MagicMock() offsets = mock.MagicMock() offsets.chunk_num = 0 offsets.num_bytes = 1 offsets.range_start = 10 sd.next_offsets.return_value = (offsets, None) dst_ase.replica_targets = [dst_ase] s._process_synccopy_descriptor(sd) assert gbr.call_count == 1 assert s._transfer_queue.qsize() == 1 assert len(s._transfer_set) == 0 # test normal append blob s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._prepare_upload = mock.MagicMock() s._process_data = mock.MagicMock() offsets = mock.MagicMock() offsets.chunk_num = 0 offsets.num_bytes = 1 offsets.range_start = 10 sd.next_offsets.return_value = (offsets, None) src_ase.mode = azmodels.StorageModes.Append s._process_synccopy_descriptor(sd) assert gbr.call_count == 2 assert s._transfer_queue.qsize() == 1 assert len(s._transfer_set) == 0 # test normal file s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._prepare_upload = mock.MagicMock() s._process_data = mock.MagicMock() offsets = mock.MagicMock() offsets.chunk_num = 0 offsets.num_bytes = 1 offsets.range_start = 10 sd.next_offsets.return_value = (offsets, None) src_ase.mode = azmodels.StorageModes.File s._process_synccopy_descriptor(sd) assert gfr.call_count == 1 assert s._transfer_queue.qsize() == 1 assert len(s._transfer_set) == 0