def test__init__custom(self): def progress_cb(progres): return upload = largetransfer.Upload(['src'], 'dst', transfer_id='xxx', description='desc', tags='tags', gzip=False, use_pigz=False, pool_size=1, chunk_size=10, progress_cb=progress_cb, cb_interval=10) assert upload.gzip is False assert upload.use_pigz is False assert upload._pool_size == 1 assert upload._chunk_size == 10 assert upload._simple is False assert upload._cb_interval == 10 assert upload._progress_cb == progress_cb assert hasattr(upload.src, '__iter__') assert upload._manifest is not None assert upload._manifest['description'] == 'desc' assert upload._manifest['tags'] == 'tags' assert upload.dst == os.path.join('dst', 'xxx') assert upload._callback_thread is None
def do_backup(op): try: self.redis_instances.save_all() dbs = [r.db_path for r in self.redis_instances if r.db_path] cloud_storage_path = bus.platform.scalrfs.backups( BEHAVIOUR) #? __node__.platform LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path) def progress_cb(progress): LOG.debug('Uploading %s bytes' % progress) uploader = largetransfer.Upload(dbs, cloud_storage_path, progress_cb=progress_cb) uploader.apply_async() uploader.join() manifest = uploader.manifest result = transfer_result_to_backup_result(manifest) node.__node__.messaging.send( DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(db_type=BEHAVIOUR, status='ok', backup_parts=result)) return result #? except (Exception, BaseException), e: LOG.exception(e) # Notify Scalr about error node.__node__.messaging.send( DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def test_apply_async3(self): """ Test apply_async method with progress callback function """ def put(*args, **kwds): time.sleep(2) cloudfs_types['file'].put = mock.MagicMock(side_effect=put) file_path, size, md5_sum = make_file() stream = open(file_path, 'rb') def progress_cb(progress): return upload = largetransfer.Upload([stream], tmp_dir, chunk_size=2, progress_cb=progress_cb) upload.apply_async() assert upload.running assert upload._callback_thread assert upload._callback_thread.is_alive() upload.join()
def test_terminate2(self): """ Test terminate method then upload is blocked """ def put(*args, **kwds): # call blocking method wait() threading.Event().wait() cloudfs_types['file'].put = mock.MagicMock(side_effect=put) file_path, size, md5_sum = make_file() stream = open(file_path, 'rb') def progress_cb(progress): return upload = largetransfer.Upload([stream], tmp_dir, chunk_size=2, progress_cb=progress_cb) upload.apply_async() # wait start time.sleep(5) assert upload.error is None, upload.error assert upload.running assert upload._callback_thread assert upload._callback_thread.is_alive() upload.terminate() assert upload.process is None assert not upload.running assert upload._callback_thread is None
def test_apply_async1(self, cloudfs_mock): file_path, size, md5_sum = make_file() stream = open(file_path, 'rb') upload = largetransfer.Upload([stream], tmp_dir, chunk_size=2) upload.apply_async() time.sleep(1) assert not upload.running
def test_progress(self): assert_flag = multiprocessing.Value('i', 0) progress_list = [] def put(*args, **kwds): for i in range(100): kwds['report_to'](i, 0) time.sleep(0.01) for i in range(100): if i not in progress_list: break else: assert_flag.value = 1 cloudfs_types['file'].put = mock.MagicMock(side_effect=put) file_path, size, md5_sum = make_file() def _on_progress(bytes_completed, size): progress_list.append(bytes_completed) upload = largetransfer.Upload([file_path], tmp_dir, simple=True) upload._on_progress = _on_progress upload.apply_async() upload.join() assert assert_flag.value == 1
def test_stop1(self): """ Test stop method when upload is not running """ upload = largetransfer.Upload('src', 'dst') upload._kill = mock.MagicMock() upload.stop() assert upload._kill.call_count == 0
def i_upload_file(context, simple): simple == 'True' sources = [source.src for source in context.sources] context.tr = largetransfer.Upload(sources, context.remote_dir, simple=simple) context.tr.apply_async() try: context.tr.join() except: context.error = sys.exc_info()
def do_backup(self): tmpdir = None dumps = [] tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp') try: # Get databases list psql = PSQL(user=self.postgresql.root_user.name) databases = psql.list_pg_databases() if 'template0' in databases: databases.remove('template0') if not os.path.exists(tmp_path): os.makedirs(tmp_path) # Dump all databases LOG.info("Dumping all databases") tmpdir = tempfile.mkdtemp(dir=tmp_path) chown_r(tmpdir, self.postgresql.root_user.name) def _single_backup(db_name): dump_path = tmpdir + os.sep + db_name + '.sql' pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path) su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args] err = system2(su_args)[1] if err: raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) # ? dumps.append(dump_path) for db_name in databases: _single_backup(db_name) cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR) suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave' backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix} LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags)) def progress_cb(progress): LOG.debug('Uploading %s bytes' % progress) uploader = largetransfer.Upload(dumps, cloud_storage_path, progress_cb=progress_cb) try: uploader.apply_async() uploader.join() manifest = uploader.manifest LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path) LOG.debug(manifest.data) return transfer_result_to_backup_result(manifest) except: uploader.terminate() raise finally: if tmpdir: shutil.rmtree(tmpdir, ignore_errors=True)
def test_terminate1(self): """ Test terminate method then upload is not running """ upload = largetransfer.Upload('src', 'dst') upload._kill = mock.MagicMock() upload.terminate() msg = 'Wrong call count for upload._kill method, expected %s, gotten %s' %\ (0, upload._kill.call_count) assert upload._kill.call_count == 0, msg
def test__init__default(self): upload = largetransfer.Upload(['src'], 'dst') assert upload.gzip is True assert upload.use_pigz is True assert upload._pool_size == largetransfer.DEFAULT_POOL_SIZE assert upload._chunk_size == largetransfer.DEFAULT_CHUNK_SIZE assert upload._simple is False assert upload._cb_interval == largetransfer.DEFAULT_CALLBACK_INTERVAL assert hasattr(upload.src, '__iter__') assert upload._manifest is not None assert upload._callback_thread is None
def test_error1(self, cloudfs_mock, queue_mock): upload = largetransfer.Upload(['src'], tmp_dir) upload._error_queue = mock.MagicMock() upload.apply_async() assert upload._error is None upload.error upload._error_queue.get.assert_called_once() upload.error upload.error msg = 'Wrong call count for upload._error_queue.get method, expected %s, gotten %s' %\ (3, upload._error_queue.get.call_count) assert upload._error_queue.get.call_count == 3, msg
def i_upload_stream_with_gzipping(context, gzip): gzip == 'True' src = [cloudfs.NamedStream(stream.src, os.path.basename(stream.src.name)) for stream in context.sources] for s in src: LOG.debug(s.name) time.sleep(1) if len(src) == 1: src = src[0] context.tr = largetransfer.Upload(src, context.remote_dir, gzip=gzip, chunk_size=5) context.tr.apply_async() try: context.tr.join() except: context.error = sys.exc_info() context.manifest = context.tr.manifest context.gzip = True
def _run(self): LOG.debug("Running MySQLDumpBackup") client = mysql_svc.MySQLClient(__mysql__['root_user'], __mysql__['root_password']) self._databases = client.list_databases() with self._run_lock: if self._killed: raise Error("Canceled") src_gen = self._gen_src() transfer_id = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') self.transfer = largetransfer.Upload(src_gen, self._dst, chunk_size=self.chunk_size, transfer_id=transfer_id) self.transfer.apply_async() self.transfer.join() result = self.transfer.manifest def log_stderr(popen): LOG.debug("mysqldump log_stderr communicate") out, err = popen.communicate() LOG.debug("mysqldump log_stderr communicate done") if err: LOG.debug("mysqldump (code %s) stderr for %s: %s", popen.returncode, popen.db_name, err) return popen.db_name, popen.returncode, err mysqldump_results = map(log_stderr, self._popens) if self._killed: raise Error("Canceled") mysqldump_errors = [] for db_name, retcode, err in mysqldump_results: if retcode: mysqldump_errors.append('%s: "%s"' % (db_name, err)) if mysqldump_errors: raise Error("Mysqldump has returned a non-zero code.\n" + '\n'.join(mysqldump_errors)) parts = transfer_result_to_backup_result(result) return backup.restore(type='mysqldump', cloudfs_source=result.cloudfs_path, parts=parts, description=self.description, tags=self.tags)
def test_error2(self, cloudfs_mock, queue_mock): def side_effect(*args, **kwds): raise Queue.Empty() upload = largetransfer.Upload(['src'], tmp_dir) upload._error_queue = mock.MagicMock() upload._error_queue.get.side_effect = side_effect upload.apply_async() assert upload._error is None upload.error upload._error_queue.get.assert_called_once() upload.error upload.error msg = 'Wrong call count for upload._error_queue.get method, expected %s, gotten %s' %\ (3, upload._error_queue.get.call_count) assert upload._error_queue.get.call_count == 3, msg
def test_join2(self): """ Test join method then upload is started """ def put(*args, **kwds): time.sleep(2) cloudfs_types['file'].put = mock.MagicMock(side_effect=put) file_path, size, md5_sum = make_file() stream = open(file_path, 'rb') upload = largetransfer.Upload([stream], tmp_dir, chunk_size=2) upload.apply_async() upload.join() assert upload.error is None assert upload.process is None
def test_final_progress(self): def put(*args, **kwds): time.sleep(5) file_path, size, md5_sum = make_file(size=10) stream = open(file_path, 'rb') cloudfs_types['file'].put = mock.MagicMock(side_effect=put) progress_cb = mock.MagicMock() upload = largetransfer.Upload([stream], tmp_dir, gzip=False, chunk_size=2, progress_cb=progress_cb) upload.apply_async() upload.join() assert upload.progress == 10 * 1024 * 1024, upload.progress assert progress_cb.call_count > 2
def upload_lvm_snapshot(self, lvm_snap, tags, path): """ Method which uploads data from lvm snapshot to cloud storage and updates snapshot status. EphVolume runs this method in separate thread """ try: self._snap_status = self.QUEUED mpoint = tempfile.mkdtemp() opts = [] if coreutils.blkid(lvm_snap.device).get('type') == 'xfs': opts += ['-o', 'nouuid,ro'] mount.mount(lvm_snap.device, mpoint, *opts) self.data_size = coreutils.statvfs(mpoint)['used'] try: cmd = ['/bin/tar', 'cp', mpoint] stream = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True).stdout src = cloudfs.NamedStream(stream, 'lvm_snapshot', streamer='tar', extension='tar') dst = path transfer = largetransfer.Upload(src, dst, tags=tags, transfer_id=self.id) self._snap_status = self.IN_PROGRESS transfer.apply_async() transfer.join() manifesto = transfer.manifest self.path = manifesto.cloudfs_path self._snap_status = self.COMPLETED finally: mount.umount(mpoint) os.rmdir(mpoint) except: self._snap_status = self.FAILED LOG.exception('Caught error while uploading LVM snapshot') finally: lvm_snap.destroy()
def test_join3(self): """ Test join method raises exception """ def put(*args, **kwds): raise Exception('Error message') cloudfs_types['file'].put = mock.MagicMock(side_effect=put) file_path, size, md5_sum = make_file() stream = open(file_path, 'rb') upload = largetransfer.Upload([stream], tmp_dir, chunk_size=2) upload.apply_async() try: upload.join() except largetransfer.TransferError as e: assert len(e.args) == 3 assert e.args[0] == 'Exception' assert e.args[1] == 'Error message' else: assert False
def do_backup(self): self.redis_instances.save_all() dbs = [r.db_path for r in self.redis_instances if r.db_path] cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR) LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path) def progress_cb(progress): LOG.debug('Uploading %s bytes' % progress) uploader = largetransfer.Upload(dbs, cloud_storage_path, progress_cb=progress_cb) try: uploader.apply_async() uploader.join() manifest = uploader.manifest return transfer_result_to_backup_result(manifest) except: uploader.terminate() raise
def i_start_upload(context, simple): simple == 'True' sources = [source.src for source in context.sources] context.tr = largetransfer.Upload(sources, context.remote_dir, simple=simple) context.tr.apply_async()
def rebundle(self): rebundle_dir = tempfile.mkdtemp() try: pl = bus.platform proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.get_storage_conn() # Determine the root filesystem size devices = coreutils.df() root_disk = firstmatched(lambda x: x.mpoint == '/', devices) if not root_disk: raise HandlerError("Can't find root device") # in bytes adjusted to 512 block device size fssize = (root_disk.size * 1000 / 512) * 512 # Old code. Should be reworked if os.path.exists('/dev/root'): root_part_path = os.path.realpath('/dev/root') else: rootfs_stat = os.stat('/') root_device_minor = os.minor(rootfs_stat.st_dev) root_device_major = os.major(rootfs_stat.st_dev) root_part_path = os.path.realpath('/dev/block/{0}:{1}'.format( root_device_major, root_device_minor)) root_part_sysblock_path = glob.glob( '/sys/block/*/%s' % os.path.basename(root_part_path))[0] root_device = '/dev/%s' % os.path.basename( os.path.dirname(root_part_sysblock_path)) arch_name = '%s.tar.gz' % self._role_name.lower() arch_path = os.path.join(rebundle_dir, arch_name) # update gcimagebundle try: pkgmgr.latest(self.gcimagebundle_pkg_name) except: e = sys.exc_info()[1] LOG.warn('Gcimagebundle update failed: %s' % e) if os_dist.redhat_family: semanage = software.which('semanage') if not semanage: pkgmgr.installed('policycoreutils-python') semanage = software.which('semanage') util.system2((semanage, 'permissive', '-a', 'rsync_t')) gc_img_bundle_bin = software.which('gcimagebundle') o, e, p = util.system2( (gc_img_bundle_bin, '-d', root_device, '-e', ','.join( self.exclude_dirs), '--fssize', str(fssize), '-o', rebundle_dir, '--output_file_name', arch_name), raise_exc=False) if p: raise HandlerError( 'Gcimagebundle util returned non-zero code %s. Stderr: %s' % (p, e)) try: LOG.info('Uploading compressed image to cloud storage') tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint( 1, 1000000), int(time.time())) remote_dir = 'gcs://%s' % tmp_bucket_name def progress_cb(progress): LOG.debug('Uploading {perc}%'.format( perc=progress / os.path.getsize(arch_path))) uploader = largetransfer.Upload(arch_path, remote_dir, simple=True, progress_cb=progress_cb) uploader.apply_async() try: try: uploader.join() except: if uploader.error: error = uploader.error[1] else: error = sys.exc_info()[1] msg = 'Image upload failed. Error:\n{error}' msg = msg.format(error=error) raise HandlerError(msg) except: with util.capture_exception(LOG): objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() cloudstorage.buckets().delete( bucket=tmp_bucket_name).execute() finally: os.unlink(arch_path) finally: shutil.rmtree(rebundle_dir) goog_image_name = self._role_name.lower().replace( '_', '-') + '-' + str(int(time.time())) try: LOG.info('Registering new image %s' % goog_image_name) compute = pl.get_compute_conn() image_url = 'http://storage.googleapis.com/%s/%s' % ( tmp_bucket_name, arch_name) req_body = dict(name=goog_image_name, sourceType='RAW', rawDisk=dict(source=image_url)) req = compute.images().insert(project=proj_id, body=req_body) operation = req.execute()['name'] LOG.info('Waiting for image to register') def image_is_ready(): req = compute.globalOperations().get(project=proj_id, operation=operation) res = req.execute() if res['status'] == 'DONE': if res.get('error'): errors = [] for e in res['error']['errors']: err_text = '%s: %s' % (e['code'], e['message']) errors.append(err_text) raise Exception('\n'.join(errors)) return True return False util.wait_until(image_is_ready, logger=LOG, timeout=600) finally: try: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() except: e = sys.exc_info()[1] LOG.error('Failed to remove image compressed source: %s' % e) return '%s/images/%s' % (proj_name, goog_image_name)
def test__init__list_src(self): upload = largetransfer.Upload(['src'], 'dst') assert hasattr(upload.src, '__iter__') assert upload._manifest is not None assert upload._simple is False assert upload._callback_thread is None
def test__init__simple(self): upload = largetransfer.Upload(['src'], 'dst', simple=True) assert hasattr(upload.src, '__iter__') assert upload._manifest is None assert upload._simple is True assert upload._callback_thread is None
def _run(self): self._check_backup_type() kwds = { 'stream': 'xbstream', # Compression is broken #'compress': True, #'compress_threads': os.sysconf('SC_NPROCESSORS_ONLN'), 'ibbackup': 'xtrabackup', 'user': __mysql__['root_user'], 'password': __mysql__['root_password'] } if self.no_lock: kwds['no_lock'] = True if not int(__mysql__['replication_master']): kwds['safe_slave_backup'] = True kwds['slave_info'] = True current_lsn = None if self.backup_type == 'auto': client = self._client() innodb_stat = client.fetchone('SHOW ENGINE INNODB STATUS')[0] #innodb_stat = client.fetchone('SHOW INNODB STATUS')[0] for line in innodb_stat.splitlines(): m = self._re_lsn_innodb_stat.search(line) if m: current_lsn = int(m.group(1)) break if self.backup_type in ('incremental', 'auto'): if self.prev_cloudfs_source: # Download manifest and get it's to_lsn mnf = cloudfs.Manifest(cloudfs_path=self.prev_cloudfs_source) self.from_lsn = mnf.meta['to_lsn'] else: self._check_attr('from_lsn') if self.backup_type == 'incremental' or \ (self.backup_type == 'auto' and current_lsn and current_lsn >= self.from_lsn): kwds.update({ 'incremental': True, 'incremental_lsn': self.from_lsn }) LOG.debug('self._config: %s', self._config) LOG.debug('kwds: %s', kwds) if self.backup_type == 'incremental': LOG.info('Creating incremental xtrabackup (from LSN: %s)', self.from_lsn) else: LOG.info('Creating full xtrabackup') with self._xbak_init_lock: if self._killed: raise Error("Canceled") self._xbak = innobackupex.args(__mysql__['tmp_dir'], **kwds).popen() gzip = self.compressor == 'gzip' transfer_id = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') self._transfer = largetransfer.Upload(self._xbak.stdout, self.cloudfs_target, gzip=gzip, transfer_id=transfer_id) stderr_thread, stderr = cloudfs.readfp_thread(self._xbak.stderr) self._transfer.apply_async() self._transfer.join() manifesto = self._transfer.manifest if self._killed: raise Error("Canceled") stderr_thread.join() self._xbak.wait() stderr = stderr[0] if stderr else '' if self._xbak.returncode: raise Error(stderr) with self._xbak_init_lock: self._xbak = None self._transfer = None log_file = log_pos = to_lsn = None re_binlog = self._re_binlog \ if int(__mysql__['replication_master']) else \ self._re_slave_binlog for line in stderr.splitlines(): m = self._re_lsn.search(line) or self._re_lsn_51.search(line) if m: to_lsn = m.group(1) continue m = re_binlog.search(line) if m: log_file = m.group(1) log_pos = int(m.group(2)) continue if log_file and log_pos and to_lsn: break rst = backup.restore(type='xtrabackup', backup_type=self.backup_type, from_lsn=self.from_lsn, to_lsn=to_lsn, cloudfs_source=manifesto.cloudfs_path, prev_cloudfs_source=self.prev_cloudfs_source, log_file=log_file, log_pos=log_pos) # Update manifest LOG.debug('rst: %s', dict(rst)) manifesto.meta = dict(rst) manifesto.save() LOG.info( 'Created %s xtrabackup. (LSN: %s..%s, log_file: %s, log_pos: %s)', rst.backup_type, rst.from_lsn, rst.to_lsn, rst.log_file, rst.log_pos) return rst
def test_not_running(self): upload = largetransfer.Upload(['src'], 'dst') assert not upload.running
def snapshot(self, op, name): rebundle_dir = tempfile.mkdtemp() archive_path = '' try: pl = __node__['platform'] proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.get_storage_conn() root_part_path = None for d in coreutils.df(): if d.mpoint == '/': root_part_path = d.device break else: raise ImageAPIError('Failed to find root volume') root_part_sysblock_path = glob.glob( '/sys/block/*/%s' % os.path.basename(root_part_path))[0] root_device = '/dev/%s' % os.path.basename( os.path.dirname(root_part_sysblock_path)) archive_name = '%s.tar.gz' % name.lower() archive_path = os.path.join(rebundle_dir, archive_name) self._prepare_software() gcimagebundle_bin = software.which('gcimagebundle') out, err, code = util.system2( (gcimagebundle_bin, '-d', root_device, '-e', ','.join( self.exclude_dirs), '-o', rebundle_dir, '--output_file_name', archive_name), raise_exc=False) if code: raise ImageAPIError( 'Gcimagebundle util returned non-zero code %s. Stderr: %s' % (code, err)) LOG.info('Uploading compressed image to cloud storage') tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint( 1, 1000000), int(time.time())) remote_dir = 'gcs://%s' % tmp_bucket_name def progress_cb(progress): LOG.debug('Uploading {perc}%'.format( perc=progress / os.path.getsize(archive_path))) uploader = largetransfer.Upload(archive_path, remote_dir, simple=True, progress_cb=progress_cb) uploader.apply_async() try: uploader.join() except: if uploader.error: error = uploader.error[1] else: error = sys.exc_info()[1] msg = 'Image upload failed. Error:\n{error}' msg = msg.format(error=error) self._remove_bucket(tmp_bucket_name, archive_name, cloudstorage) raise ImageAPIError(msg) finally: shutil.rmtree(rebundle_dir) if os.path.exists(archive_path): os.remove(archive_path) image_name = name.lower().replace('_', '-') + '-' + str( int(time.time())) self._register_image(image_name, tmp_bucket_name, archive_name, cloudstorage) return '%s/images/%s' % (proj_name, image_name)
def do_backup(op): tmpdir = None dumps = [] tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp') try: # Get databases list psql = PSQL(user=self.postgresql.root_user.name) databases = psql.list_pg_databases() if 'template0' in databases: databases.remove('template0') if not os.path.exists(tmp_path): os.makedirs(tmp_path) # Dump all databases LOG.info("Dumping all databases") tmpdir = tempfile.mkdtemp(dir=tmp_path) chown_r(tmpdir, self.postgresql.root_user.name) def _single_backup(db_name): dump_path = tmpdir + os.sep + db_name + '.sql' pg_args = '%s %s --no-privileges -f %s' % ( PG_DUMP, db_name, dump_path) su_args = [ SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args ] err = system2(su_args)[1] if err: raise HandlerError( 'Error while dumping database %s: %s' % (db_name, err)) #? dumps.append(dump_path) for db_name in databases: _single_backup(db_name) cloud_storage_path = __node__.platform.scalrfs.backups( BEHAVIOUR) suffix = 'master' if int( __postgresql__[OPT_REPLICATION_MASTER]) else 'slave' backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix} LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags)) def progress_cb(progress): LOG.debug('Uploading %s bytes' % progress) uploader = largetransfer.Upload(dumps, cloud_storage_path, progress_cb=progress_cb) uploader.apply_async() uploader.join() manifest = uploader.manifest LOG.info( "Postgresql backup uploaded to cloud storage under %s", cloud_storage_path) LOG.debug(manifest.data) # Notify Scalr result = transfer_result_to_backup_result(manifest) __node__.messaging.send( DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(db_type=BEHAVIOUR, status='ok', backup_parts=result)) return result except (Exception, BaseException), e: LOG.exception(e) # Notify Scalr about error __node__.messaging.send( DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def test_join1(self): """ Test join method then upload is not started """ upload = largetransfer.Upload('src', 'dst') upload.join()
def test_initial_progress(self): upload = largetransfer.Upload('src', 'dst') assert upload.progress == 0