def _run(self):
LOG.debug("Running MySQLDumpBackup")
client = mysql_svc.MySQLClient(__mysql__['root_user'],
__mysql__['root_password'])
self._databases = client.list_databases()
with self._run_lock:
if self._killed:
raise Error("Canceled")
self.transfer = cloudfs.LargeTransfer(self._gen_src,
self._dst,
streamer=None,
chunk_size=self.chunk_size)
result = self.transfer.run()
if not result:
raise Error("Error while transfering to cloud storage")
def log_stderr(popen):
LOG.debug("mysqldump log_stderr communicate")
out, err = popen.communicate()
LOG.debug("mysqldump log_stderr communicate done")
if err:
LOG.debug("mysqldump stderr: %s", err)
map(log_stderr, self._popens)
if self._killed:
raise Error("Canceled")
result = transfer_result_to_backup_result(result)
return result
def do_backup(op):
try:
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances if r.db_path]
cloud_storage_path = bus.platform.scalrfs.backups(
BEHAVIOUR) #? __node__.platform
LOG.info("Uploading backup to cloud storage (%s)",
cloud_storage_path)
transfer = LargeTransfer(dbs, cloud_storage_path)
result = transfer.run()
result = transfer_result_to_backup_result(result)
node.__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
return result #?
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
node.__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def _run(self):
LOG.debug("Running MySQLDumpBackup")
client = mysql_svc.MySQLClient(
__mysql__['root_user'],
__mysql__['root_password'])
self._databases = client.list_databases()
with self._run_lock:
if self._killed:
raise Error("Canceled")
self.transfer = cloudfs.LargeTransfer(self._gen_src, self._dst,
streamer=None, chunk_size=self.chunk_size)
result = self.transfer.run()
if not result:
raise Error("Error while transfering to cloud storage")
def log_stderr(popen):
LOG.debug("mysqldump log_stderr communicate")
out, err = popen.communicate()
LOG.debug("mysqldump log_stderr communicate done")
if err:
LOG.debug("mysqldump stderr: %s", err)
map(log_stderr, self._popens)
if self._killed:
raise Error("Canceled")
result = transfer_result_to_backup_result(result)
return result
def do_backup(op):
try:
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances if r.db_path]
cloud_storage_path = bus.platform.scalrfs.backups(
BEHAVIOUR) #? __node__.platform
LOG.info("Uploading backup to cloud storage (%s)",
cloud_storage_path)
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dbs,
cloud_storage_path,
progress_cb=progress_cb)
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
result = transfer_result_to_backup_result(manifest)
node.__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
return result #?
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
node.__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def do_backup(op):
try:
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances if r.db_path]
cloud_storage_path = bus.platform.scalrfs.backups(BEHAVIOUR) #? __node__.platform
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
transfer = LargeTransfer(dbs, cloud_storage_path)
result = transfer.run()
result = transfer_result_to_backup_result(result)
node.__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result #?
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
node.__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
trn = LargeTransfer(dumps, cloud_storage_path, tags=backup_tags)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
def do_backup(self):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) # ?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps, cloud_storage_path, progress_cb=progress_cb)
try:
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
LOG.debug(manifest.data)
return transfer_result_to_backup_result(manifest)
except:
uploader.terminate()
raise
finally:
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
def test_transfer_result_to_backup_result():
mnf = mock.Mock(files=[{
'chunks': [['f101.part', None, 1048576], ['f102.part', None, 5942]]
}, {
'chunks': [['f201.part', None, 65403]]
}],
cloudfs_path='s3://path/to/backup/manifest.json')
result = handlers.transfer_result_to_backup_result(mnf)
assert len(result) == 3
assert result[0] == dict(path='s3://path/to/backup/f101.part',
size=1048576)
assert result[1] == dict(path='s3://path/to/backup/f102.part', size=5942)
assert result[2] == dict(path='s3://path/to/backup/f201.part', size=65403)
def _run(self):
LOG.debug("Running MySQLDumpBackup")
client = mysql_svc.MySQLClient(__mysql__['root_user'],
__mysql__['root_password'])
self._databases = client.list_databases()
with self._run_lock:
if self._killed:
raise Error("Canceled")
src_gen = self._gen_src()
transfer_id = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S')
self.transfer = largetransfer.Upload(src_gen,
self._dst,
chunk_size=self.chunk_size,
transfer_id=transfer_id)
self.transfer.apply_async()
self.transfer.join()
result = self.transfer.manifest
def log_stderr(popen):
LOG.debug("mysqldump log_stderr communicate")
out, err = popen.communicate()
LOG.debug("mysqldump log_stderr communicate done")
if err:
LOG.debug("mysqldump (code %s) stderr for %s: %s",
popen.returncode, popen.db_name, err)
return popen.db_name, popen.returncode, err
mysqldump_results = map(log_stderr, self._popens)
if self._killed:
raise Error("Canceled")
mysqldump_errors = []
for db_name, retcode, err in mysqldump_results:
if retcode:
mysqldump_errors.append('%s: "%s"' % (db_name, err))
if mysqldump_errors:
raise Error("Mysqldump has returned a non-zero code.\n" +
'\n'.join(mysqldump_errors))
parts = transfer_result_to_backup_result(result)
return backup.restore(type='mysqldump',
cloudfs_source=result.cloudfs_path,
parts=parts,
description=self.description,
tags=self.tags)
def _run(self):
LOG.debug("Running MySQLDumpBackup")
client = mysql_svc.MySQLClient(__mysql__['root_user'],
__mysql__['root_password'])
self._databases = client.list_databases()
with self._run_lock:
if self._killed:
raise Error("Canceled")
self.transfer = cloudfs.LargeTransfer(self._gen_src,
self._dst,
streamer=None,
chunk_size=self.chunk_size)
result = self.transfer.run()
if not result:
raise Error("Error while transfering to cloud storage")
def log_stderr(popen):
LOG.debug("mysqldump log_stderr communicate")
out, err = popen.communicate()
LOG.debug("mysqldump log_stderr communicate done")
if err:
LOG.debug("mysqldump (code %s) stderr for %s: %s",
popen.returncode, popen.db_name, err)
return popen.db_name, popen.returncode, err
mysqldump_results = map(log_stderr, self._popens)
if self._killed:
raise Error("Canceled")
mysqldump_errors = []
for db_name, retcode, err in mysqldump_results:
if retcode:
mysqldump_errors.append('%s: "%s"' % (db_name, err))
if mysqldump_errors:
raise Error("Mysqldump has returned a non-zero code.\n" +
'\n'.join(mysqldump_errors))
parts = transfer_result_to_backup_result(result)
return backup.restore(type='mysqldump',
cloudfs_source=result.cloudfs_path,
parts=parts,
description=self.description,
tags=self.tags)
def on_DbMsr_CreateBackup(self, message):
try:
op = operation(name=self._op_backup, phases=[{
'name': self._phase_backup,
'steps': [self._step_copy_database_file,
self._step_upload_to_cloud_storage]
}])
op.define()
with op.phase(self._phase_backup):
with op.step(self._step_copy_database_file):
# Flush redis data on disk before creating backup
LOG.info("Dumping Redis data on disk")
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances]
with op.step(self._step_upload_to_cloud_storage):
cloud_storage_path = self._platform.scalrfs.backups(BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
transfer = LargeTransfer(dbs, cloud_storage_path)
result = transfer.run()
result = handlers.transfer_result_to_backup_result(result)
op.ok(data=result)
# Notify Scalr
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'ok',
backup_parts = result
))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'error',
last_error = str(e)
))
def on_DbMsr_CreateBackup(self, message):
try:
op = operation(name=self._op_backup, phases=[{
'name': self._phase_backup,
'steps': [self._step_copy_database_file,
self._step_upload_to_cloud_storage]
}])
op.define()
with op.phase(self._phase_backup):
with op.step(self._step_copy_database_file):
# Flush redis data on disk before creating backup
LOG.info("Dumping Redis data on disk")
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances]
with op.step(self._step_upload_to_cloud_storage):
cloud_storage_path = self._platform.scalrfs.backups(BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
transfer = LargeTransfer(dbs, cloud_storage_path)
result = transfer.run()
result = handlers.transfer_result_to_backup_result(result)
op.ok(data=result)
# Notify Scalr
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'ok',
backup_parts = result
))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'error',
last_error = str(e)
))
def _run(self):
LOG.debug("Running MySQLDumpBackup")
client = mysql_svc.MySQLClient(
__mysql__['root_user'],
__mysql__['root_password'])
self._databases = client.list_databases()
with self._run_lock:
if self._killed:
raise Error("Canceled")
self.transfer = cloudfs.LargeTransfer(self._gen_src, self._dst,
streamer=None, chunk_size=self.chunk_size)
result = self.transfer.run()
if not result:
raise Error("Error while transfering to cloud storage")
def log_stderr(popen):
LOG.debug("mysqldump log_stderr communicate")
out, err = popen.communicate()
LOG.debug("mysqldump log_stderr communicate done")
if err:
LOG.debug("mysqldump (code %s) stderr for %s: %s", popen.returncode, popen.db_name, err)
return popen.db_name, popen.returncode, err
mysqldump_results = map(log_stderr, self._popens)
if self._killed:
raise Error("Canceled")
mysqldump_errors = []
for db_name, retcode, err in mysqldump_results:
if retcode:
mysqldump_errors.append('%s: "%s"' % (db_name, err))
if mysqldump_errors:
raise Error("Mysqldump has returned a non-zero code.\n" +
'\n'.join(mysqldump_errors))
parts = transfer_result_to_backup_result(result)
return backup.restore(type='mysqldump',
cloudfs_source=result.cloudfs_path,
parts=parts,
description=self.description,
tags=self.tags)
def do_backup(self):
self.redis_instances.save_all()
dbs = [r.db_path for r in self.redis_instances if r.db_path]
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dbs,
cloud_storage_path,
progress_cb=progress_cb)
try:
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
return transfer_result_to_backup_result(manifest)
except:
uploader.terminate()
raise
def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (
PG_DUMP, db_name, dump_path)
su_args = [
SU_EXEC, '-', self.postgresql.root_user.name, '-c',
pg_args
]
err = system2(su_args)[1]
if err:
raise HandlerError(
'Error while dumping database %s: %s' %
(db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(
BEHAVIOUR)
suffix = 'master' if int(
__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" %
(cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps,
cloud_storage_path,
progress_cb=progress_cb)
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info(
"Postgresql backup uploaded to cloud storage under %s",
cloud_storage_path)
LOG.debug(manifest.data)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))