def _create_downloader_for_start(td): d = ops.Downloader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) d._general_options.dry_run = False d._cleanup_temporary_files = mock.MagicMock() d._download_start = datetime.datetime.now(tz=dateutil.tz.tzlocal()) d._initialize_transfer_threads = mock.MagicMock() d._general_options.concurrency.crypto_processes = 1 d._general_options.concurrency.md5_processes = 1 d._general_options.concurrency.disk_threads = 1 d._general_options.concurrency.transfer_threads = 1 d._general_options.resume_file = pathlib.Path(str(td.join('rf'))) d._spec.sources = [] d._spec.options = mock.MagicMock() d._spec.options.chunk_size_bytes = 1 d._spec.options.mode = azmodels.StorageModes.Auto d._spec.options.overwrite = True d._spec.options.rename = False d._spec.options.strip_components = 0 d._spec.skip_on = mock.MagicMock() d._spec.skip_on.md5_match = False d._spec.skip_on.lmt_ge = False d._spec.skip_on.filesize_match = False d._spec.destination = mock.MagicMock() d._spec.destination.path = pathlib.Path(str(td)) d._download_start_time = util.datetime_now() d._pre_md5_skip_on_check = mock.MagicMock() d._check_download_conditions = mock.MagicMock() d._all_remote_files_processed = False p = '/cont/remote/path' asp = azops.SourcePath() asp.add_path_with_storage_account(p, 'sa') d._spec.sources.append(asp) return d
def test_run(srm, gbr, gfr): s = ops.SyncCopy(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) s._general_options.dry_run = False s._general_options.concurrency.transfer_threads = 1 s._general_options.resume_file = 'resume' s._spec.options.chunk_size_bytes = 0 src_ase = mock.MagicMock() src_ase._client.primary_endpoint = 'ep' src_ase.container = 'srccont' src_ase.path = 'srcasepath' src_ase.mode = azmodels.StorageModes.Block src_ase.size = 10 src_ase.name = 'srcase' dst_ase = mock.MagicMock() dst_ase._client.primary_endpoint = 'ep2' dst_ase.container = 'dstcont' dst_ase.path = 'dstasepath' dst_ase.mode = azmodels.StorageModes.Block dst_ase.size = 10 dst_ase.name = 'dstase' dst_ase.from_local = False dst_ase.replica_targets = None s._bind_sources_to_destination = mock.MagicMock() s._bind_sources_to_destination.return_value = [(src_ase, dst_ase)] s._prepare_upload = mock.MagicMock() s._put_data = mock.MagicMock() s._finalize_upload = mock.MagicMock() # normal execution s._synccopy_start_time = (util.datetime_now() - datetime.timedelta(seconds=1)) s._run() assert s._prepare_upload.call_count == 1 assert s._put_data.call_count == 1 # dry run s._general_options.dry_run = True s._run() assert s._prepare_upload.call_count == 1 assert s._put_data.call_count == 1 s._general_options.dry_run = False # replica targets with mismatch s._synccopy_start_time = None dst_ase.replica_targets = [dst_ase] with pytest.raises(RuntimeError): s._run() # exception during worker thread dst_ase.replica_targets = None with pytest.raises(RuntimeError): s._process_synccopy_descriptor = mock.MagicMock() s._process_synccopy_descriptor.side_effect = RuntimeError() s._run()
def test_update_progress_bar(): go = mock.MagicMock() go.progress_bar = True go.log_file = 'abc' start = util.datetime_now() ops.update_progress_bar( go, 'download', start, None, 1, None, 1) ops.update_progress_bar( go, 'upload', start, 1, 0, 256, 0, stdin_upload=True) with mock.patch('blobxfer.util.datetime_now') as patched_dt: patched_dt.return_value = start ops.update_progress_bar( go, 'synccopy', start, 1, 1, 1, 1) assert util.is_not_empty(go.log_file)
def test_restore_file_lmt(tmpdir): lp = pathlib.Path(str(tmpdir.join('a'))) lp.touch(mode=0o666, exist_ok=False) lp.exists() ts = util.datetime_now() - datetime.timedelta(seconds=60) ts_posix = time.mktime(ts.timetuple()) stat = lp.stat() assert stat.st_mtime != ts_posix opts = mock.MagicMock() opts.check_file_md5 = True opts.chunk_size_bytes = 16 opts.restore_file_properties.lmt = True ase = azmodels.StorageEntity('cont') ase._size = 32 ase._lmt = ts d = models.Descriptor(lp, ase, opts, mock.MagicMock(), None) d._restore_file_lmt() stat = lp.stat() assert stat.st_mtime == ts_posix
def test_run(lfmo, urm, tmpdir): u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) u._general_options.dry_run = False u._initialize_disk_threads = mock.MagicMock() u._initialize_transfer_threads = mock.MagicMock() u._general_options.concurrency.disk_threads = 1 u._general_options.concurrency.transfer_threads = 1 u._general_options.concurrency.md5_processes = 1 u._general_options.concurrency.crypto_processes = 1 u._general_options.resume_file = 'resume' u._spec.options.store_file_properties.md5 = True u._spec.skip_on.md5_match = True u._spec.options.rsa_public_key = 'abc' u._spec.options.chunk_size_bytes = 0 u._spec.options.one_shot_bytes = 0 u._spec.options.delete_only = False # check rename failure u._spec.sources.can_rename.return_value = False u._spec.options.rename = True with pytest.raises(RuntimeError): u._run() u._upload_terminate = True assert urm.call_count == 0 assert lfmo.call_count == 0 assert lfmo.initialize_check_thread.call_count == 0 assert u._initialize_disk_threads.call_count == 0 assert u._initialize_transfer_threads.call_count == 0 # check dupe u._spec.sources.can_rename.return_value = False u._spec.options.rename = False ase = mock.MagicMock() ase.client.primary_endpoint = 'ep' ase._client.primary_endpoint = 'ep' ase.path = 'asepath' ase.mode = azmodels.StorageModes.Block ase.is_encrypted = False ase.replica_targets = None ase.container = 'cont' ase.name = 'name' ase.size = 10 ase2 = mock.MagicMock() ase2.client.primary_endpoint = 'ep2' ase2._client.primary_endpoint = 'ep2' ase2.path = 'asepath2' ase2.mode = azmodels.StorageModes.Block ase2.is_encrypted = False ase2.replica_targets = None ase2.container = 'cont2' ase2.name = 'name2' ase2.size = 10 sa = mock.MagicMock() sa.name = 'name' sa.endpoint = 'ep' sa.block_blob_client.primary_endpoint = 'pep' tmpdir.join('a').write('z' * 10) lp = mock.MagicMock() lp.relative_path = pathlib.Path('a') lp.absolute_path = pathlib.Path(str(tmpdir.join('a'))) lp.view.fd_start = 0 lp.view.fd_end = 10 lp.use_stdin = False lp.size = 10 lp.total_size = 10 u._generate_destination_for_source = mock.MagicMock() with pytest.raises(RuntimeError): u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase)] u._spec.sources.files.return_value = [lp] u._run() u._upload_terminate = True assert urm.call_count == 1 assert lfmo.call_count == 1 assert u._md5_offload.initialize_check_thread.call_count == 1 assert u._initialize_disk_threads.call_count == 1 assert u._initialize_transfer_threads.call_count == 1 # mismatch exception raise u._spec.options.vectored_io.distribution_mode = \ models.VectoredIoDistributionMode.Disabled u._check_upload_conditions = mock.MagicMock() u._check_upload_conditions.return_value = ops.UploadAction.Skip u._generate_destination_for_source.return_value = [(sa, ase)] u._spec.sources.files.return_value = [lp] with pytest.raises(RuntimeError): u._run() u._upload_terminate = True u._check_upload_conditions.return_value = ops.UploadAction.CheckMd5 with pytest.raises(RuntimeError): u._run() u._upload_terminate = True # regular execution u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) u._general_options.dry_run = False u._general_options.concurrency.disk_threads = 1 u._general_options.concurrency.transfer_threads = 1 u._general_options.concurrency.md5_processes = 1 u._general_options.concurrency.crypto_processes = 0 u._general_options.resume_file = 'resume' u._spec.options.store_file_properties.md5 = True u._spec.skip_on.md5_match = True u._spec.options.rsa_public_key = None u._spec.options.chunk_size_bytes = 0 u._spec.options.one_shot_bytes = 0 u._spec.sources.can_rename.return_value = False u._spec.options.rename = False u._spec.options.delete_only = False u._spec.options.vectored_io.distribution_mode = \ models.VectoredIoDistributionMode.Replica u._check_upload_conditions = mock.MagicMock() u._check_upload_conditions.return_value = ops.UploadAction.Upload u._generate_destination_for_source = mock.MagicMock() u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)] u._spec.sources.files.return_value = [lp] u._put_data = mock.MagicMock() u._finalize_upload = mock.MagicMock() u._upload_start_time = (util.datetime_now() - datetime.timedelta(seconds=1)) u._run() assert u._finalize_upload.call_count == 1 # regular execution, skip dry run u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) u._general_options.dry_run = True u._general_options.concurrency.disk_threads = 1 u._general_options.concurrency.transfer_threads = 1 u._general_options.concurrency.md5_processes = 1 u._general_options.concurrency.crypto_processes = 0 u._general_options.resume_file = 'resume' u._spec.options.store_file_properties.md5 = True u._spec.skip_on.md5_match = True u._spec.options.rsa_public_key = None u._spec.options.chunk_size_bytes = 0 u._spec.options.one_shot_bytes = 0 u._spec.sources.can_rename.return_value = False u._spec.options.rename = False u._spec.options.delete_only = False u._spec.options.vectored_io.distribution_mode = \ models.VectoredIoDistributionMode.Replica u._check_upload_conditions = mock.MagicMock() u._check_upload_conditions.return_value = ops.UploadAction.Skip u._generate_destination_for_source = mock.MagicMock() u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)] u._spec.sources.files.return_value = [lp] u._put_data = mock.MagicMock() u._finalize_upload = mock.MagicMock() u._upload_start_time = (util.datetime_now() - datetime.timedelta(seconds=1)) u._run() assert u._finalize_upload.call_count == 0 # delete only u._spec.options.delete_extraneous_destination = True u._spec.options.delete_only = True u._run() assert u._finalize_upload.call_count == 0 # regular execution, upload dry run u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) u._general_options.dry_run = True u._general_options.concurrency.disk_threads = 1 u._general_options.concurrency.transfer_threads = 1 u._general_options.concurrency.md5_processes = 1 u._general_options.concurrency.crypto_processes = 0 u._general_options.resume_file = 'resume' u._spec.options.store_file_properties.md5 = True u._spec.skip_on.md5_match = True u._spec.options.rsa_public_key = None u._spec.options.chunk_size_bytes = 0 u._spec.options.one_shot_bytes = 0 u._spec.sources.can_rename.return_value = False u._spec.options.rename = False u._spec.options.delete_only = False u._spec.options.vectored_io.distribution_mode = \ models.VectoredIoDistributionMode.Replica u._check_upload_conditions = mock.MagicMock() u._check_upload_conditions.return_value = ops.UploadAction.Upload u._generate_destination_for_source = mock.MagicMock() u._generate_destination_for_source.return_value = [(sa, ase), (sa, ase2)] u._spec.sources.files.return_value = [lp] u._put_data = mock.MagicMock() u._finalize_upload = mock.MagicMock() u._upload_start_time = (util.datetime_now() - datetime.timedelta(seconds=1)) u._run() assert u._finalize_upload.call_count == 0 # exception raise u = ops.Uploader(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) u._general_options.dry_run = False u._general_options.concurrency.disk_threads = 1 u._general_options.concurrency.transfer_threads = 1 u._general_options.concurrency.md5_processes = 1 u._general_options.concurrency.crypto_processes = 0 u._general_options.resume_file = 'resume' u._spec.options.store_file_properties.md5 = True u._spec.skip_on.md5_match = True u._spec.options.rsa_public_key = None u._spec.options.chunk_size_bytes = 0 u._spec.options.one_shot_bytes = 0 u._spec.sources.can_rename.return_value = False u._spec.options.rename = False u._spec.options.delete_only = False u._spec.options.vectored_io.distribution_mode = \ models.VectoredIoDistributionMode.Disabled u._check_upload_conditions = mock.MagicMock() u._check_upload_conditions.return_value = ops.UploadAction.Upload u._generate_destination_for_source = mock.MagicMock() u._generate_destination_for_source.return_value = [(sa, ase)] u._spec.sources.files.return_value = [lp] with pytest.raises(RuntimeError): u._process_upload_descriptor = mock.MagicMock() u._process_upload_descriptor.side_effect = RuntimeError() u._run() u._upload_terminate = True