def _update_data(self, path, src): debug("Updating remote file: %s" % repr(path)) total_bytes_written = self.bytes_written bytes_written = 0 info = src.get_info() def __callback(status): bytes_written = int(status.resumable_progress) self.bytes_written = total_bytes_written + bytes_written progress = Progress(GsyncOptions.progress, __callback) if GsyncOptions.dry_run: bytes_written = info.fileSize progress(MediaUploadProgress(bytes_written, bytes_written)) else: progress.bytesTotal = info.fileSize drive = Drive() info = drive.update( path, info, media_body=src.get_uploader(), progress_callback=progress ) if info is not None: bytes_written = long(info.get('fileSize', '0')) debug("Final file size: %d" % bytes_written) else: debug("Update failed") progress.complete(bytes_written) self.bytes_written = total_bytes_written + bytes_written
def _updateFile(self, path, src): debug("Updating remote file: %s" % repr(path)) totalBytesWritten = self.bytesWritten bytesWritten = 0 info = src.getInfo() def _callback(status): bytesWritten = int(status.resumable_progress) self.bytesWritten = totalBytesWritten + bytesWritten progress = Progress(GsyncOptions.progress, _callback) if GsyncOptions.dry_run: bytesWritten = info.fileSize progress(MediaUploadProgress(bytesWritten, bytesWritten)) else: drive = Drive() info = drive.update(path, info, src.getUploader(), progress) if info is not None: bytesWritten = long(info.get('fileSize', '0')) else: debug("Update failed") progress.complete(bytesWritten) self.bytesWritten = totalBytesWritten + bytesWritten
def _update_data(self, path, src): debug("Updating remote file: %s" % repr(path)) total_bytes_written = self.bytes_written bytes_written = 0 info = src.get_info() def __callback(status): bytes_written = int(status.resumable_progress) self.bytes_written = total_bytes_written + bytes_written progress = Progress(GsyncOptions.progress, __callback) if GsyncOptions.dry_run: bytes_written = info.fileSize progress(MediaUploadProgress(bytes_written, bytes_written)) else: progress.bytesTotal = info.fileSize drive = Drive() info = drive.update(path, info, media_body=src.get_uploader(), progress_callback=progress) if info is not None: bytes_written = int(info.get('fileSize', '0')) debug("Final file size: %d" % bytes_written) else: debug("Update failed") progress.complete(bytes_written) self.bytes_written = total_bytes_written + bytes_written
def _update_data(self, path, src): path = self.get_path(path) self.get_info(path) debug("Updating local file %s" % repr(path)) uploader = src.get_uploader() fd = None bytes_written = 0 chunk_size = uploader.chunksize() file_size = uploader.size() try: if not GsyncOptions.dry_run: fd = open(path, "wb") progress = Progress(GsyncOptions.progress) while bytes_written < file_size: chunk = uploader.getbytes(bytes_written, chunk_size) debug("len(chunk) = %d" % len(chunk)) if not chunk: break if fd is not None: fd.write(chunk) chunk_len = len(chunk) bytes_written += chunk_len self.bytes_written += chunk_len progress(MediaUploadProgress(bytes_written, file_size)) debug(" Written %d bytes" % bytes_written) progress.complete(bytes_written) if bytes_written < file_size: # pragma: no cover raise Exception("Got %d bytes, expected %d bytes" % ( bytes_written, file_size )) except KeyboardInterrupt: # pragma: no cover debug("Interrupted") raise except Exception as ex: # pragma: no cover debug("Write failed: %s" % repr(ex)) raise finally: if fd is not None: fd.close()
def _update_data(self, path, src): path = self.get_path(path) self.get_info(path) debug("Updating local file %s" % repr(path)) uploader = src.get_uploader() fd = None bytes_written = 0 chunk_size = uploader.chunksize() file_size = uploader.size() try: if not GsyncOptions.dry_run: fd = open(path, "w") progress = Progress(GsyncOptions.progress) while bytes_written < file_size: chunk = uploader.getbytes(bytes_written, chunk_size) debug("len(chunk) = %d" % len(chunk)) if not chunk: break if fd is not None: fd.write(chunk) chunk_len = len(chunk) bytes_written += chunk_len self.bytes_written += chunk_len progress(MediaUploadProgress(bytes_written, file_size)) debug(" Written %d bytes" % bytes_written) progress.complete(bytes_written) if bytes_written < file_size: # pragma: no cover raise Exception("Got %d bytes, expected %d bytes" % ( bytes_written, file_size )) except KeyboardInterrupt: # pragma: no cover debug("Interrupted") raise except Exception, ex: # pragma: no cover debug("Write failed: %s" % repr(ex)) raise
def test_zero_byte_file(self): channel = Progress() self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue()) import re pat = re.compile( r'^\s+%d\s+%d%%\s+\d+\.\d{2}(?:B|KB|MB|GB|TB)/s\s+\d+:\d+:\d+$' % (0, 100), re.S | re.M) sys.stdout.truncate(0) channel.complete(0) self.assertIsNotNone(pat.search(sys.stdout.getvalue()))
def _updateFile(self, path, src): path = self.getPath(path) info = self.getInfo(path) debug("Updating local file %s" % repr(path)) uploader = src.getUploader() f = None bytesWritten = 0 chunkSize = uploader.chunksize() fileSize = uploader.size() try: if not GsyncOptions.dry_run: f = open(path, "w") progress = Progress(GsyncOptions.progress) while bytesWritten < fileSize: chunk = uploader.getbytes(bytesWritten, chunkSize) debug("len(chunk) = %d" % len(chunk)) if not chunk: break if f is not None: f.write(chunk) chunkLen = len(chunk) bytesWritten += chunkLen self.bytesWritten += chunkLen progress(MediaUploadProgress(bytesWritten, fileSize)) debug(" Written %d bytes" % bytesWritten) progress.complete(bytesWritten) if bytesWritten < fileSize: raise Exception("Got %d bytes, expected %d bytes" % ( bytesWritten, fileSize )) except KeyboardInterrupt: debug("Interrupted") raise except Exception, e: debug("Write failed: %s" % repr(e)) raise
def test_zero_byte_file(self): channel = Progress() self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue()) import re pat = re.compile( r'^\s+%d\s+%d%%\s+\d+\.\d{2}(?:B|KB|MB|GB|TB)/s\s+\d+:\d+:\d+$' % (0, 100), re.S | re.M ) sys.stdout.truncate(0) channel.complete(0) self.assertIsNotNone(pat.search(sys.stdout.getvalue()))
def test_status_messages_with_callback(self): def callback(status): callback.called = True callback.called = False channel = Progress(callback=callback) self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue()) import re for i in ( 5, 10, 20, 40, 50, 75, 100 ): pat = re.compile( r'^\s+%d\s+%d%%\s+\d+\.\d{2}(?:B|KB|MB|GB|TB)/s\s+\d+:\d+:\d+$' % (i, i), re.S | re.M ) sys.stdout.truncate(0) channel(ProgressStatus(100, i)) self.assertIsNotNone(pat.search(sys.stdout.getvalue())) self.assertTrue(callback.called)
def _updateFile(self, path, src): path = self.getPath(path) info = self.getInfo(path) debug("Updating local file %s" % repr(path)) uploader = src.getUploader() f = None bytesWritten = 0 chunkSize = uploader.chunksize() fileSize = uploader.size() try: if not GsyncOptions.dry_run: f = open(path, "w") progress = Progress(GsyncOptions.progress) while bytesWritten < fileSize: chunk = uploader.getbytes(bytesWritten, chunkSize) debug("len(chunk) = %d" % len(chunk)) if not chunk: break if f is not None: f.write(chunk) chunkLen = len(chunk) bytesWritten += chunkLen self.bytesWritten += chunkLen progress(MediaUploadProgress(bytesWritten, fileSize)) debug(" Written %d bytes" % bytesWritten) progress.complete(bytesWritten) if bytesWritten < fileSize: raise Exception("Got %d bytes, expected %d bytes" % (bytesWritten, fileSize)) except KeyboardInterrupt: debug("Interrupted") raise except Exception, e: debug("Write failed: %s" % repr(e)) raise
def test_rate_normalization(self): channel = Progress() self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue()) fileSize = 1000000000 import re pat = re.compile( r'^\s+%d\s+%d%%\s+\d+\.\d{2}(?:KB|MB|GB|TB)/s\s+\d+:\d+:\d+$' % (fileSize, 100), re.S | re.M) sys.stdout.truncate(0) channel(ProgressStatus(fileSize, fileSize / 4)) self.assertIsNone(pat.search(sys.stdout.getvalue())) sys.stdout.truncate(0) channel.complete(fileSize) self.assertIsNotNone(pat.search(sys.stdout.getvalue()))
def test_rate_normalization(self): channel = Progress() self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue()) fileSize = 1000000000 import re pat = re.compile( r'^\s+%d\s+%d%%\s+\d+\.\d{2}(?:KB|MB|GB|TB)/s\s+\d+:\d+:\d+$' % (fileSize, 100), re.S | re.M ) sys.stdout.truncate(0) channel(ProgressStatus(fileSize, fileSize / 4)) self.assertIsNone(pat.search(sys.stdout.getvalue())) sys.stdout.truncate(0) channel.complete(fileSize) self.assertIsNotNone(pat.search(sys.stdout.getvalue()))
def test_with_enabled_output(self): channel = Progress(enable_output=True) self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue())
def test_enabled_output_by_default(self): channel = Progress() self.assertNotEqual("", sys.stdout.getvalue()) self.assertEqual("", sys.stderr.getvalue())