def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
trn = LargeTransfer(dumps, cloud_storage_path, tags=backup_tags)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
def do_backup(self):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) # ?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps, cloud_storage_path, progress_cb=progress_cb)
try:
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
LOG.debug(manifest.data)
return transfer_result_to_backup_result(manifest)
except:
uploader.terminate()
raise
finally:
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
def _create_snapshot(self):
self._logger.info("Creating PostgreSQL data bundle")
psql = PSQL()
if self.postgresql.service.running:
psql.start_backup()
system2('sync', shell=True)
# Creating storage snapshot
snap = self._create_storage_snapshot()
if self.postgresql.service.running:
psql.stop_backup()
wait_until(lambda: snap.state in (Snapshot.CREATED, Snapshot.COMPLETED, Snapshot.FAILED))
if snap.state == Snapshot.FAILED:
raise HandlerError('postgresql storage snapshot creation failed. See log for more details')
self._logger.info('PostgreSQL data bundle created\n snapshot: %s', snap.id)
return snap
def on_DbMsr_CreateBackup(self, message):
#TODO: Think how to move the most part of it into Postgresql class
# Retrieve password for scalr pg user
tmpdir = backup_path = None
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
op = operation(name=self._op_backup, phases=[{
'name': self._phase_backup
}])
op.define()
with op.phase(self._phase_backup):
if not os.path.exists(self._tmp_path):
os.makedirs(self._tmp_path)
# Defining archive name and path
backup_filename = time.strftime('%Y-%m-%d-%H:%M:%S')+'.tar.gz'
backup_path = os.path.join(self._tmp_path, backup_filename)
# Creating archive
backup = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=self._tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err))
backup.add(dump_path, os.path.basename(dump_path))
make_backup_steps(databases, op, _single_backup)
backup.close()
with op.step(self._step_upload_to_cloud_storage):
# Creating list of full paths to archive chunks
#if os.path.getsize(backup_path) > __postgresql__['pgdump_chunk_size']:
# parts = [os.path.join(tmpdir, file) for file in split(backup_path, backup_filename, __postgresql__['pgdump_chunk_size'], tmpdir)]
#else:
# parts = [backup_path]
#sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = self._platform.scalrfs.backups(BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
trn = LargeTransfer(backup_path, cloud_storage_path)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(dict(path=os.path.join(cloud_storage_path, c[0]), size=c[2]) for c in
manifest['files'][0]['chunks'])
op.ok(data=result)
# Notify Scalr
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'ok',
backup_parts = result
))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'error',
last_error = str(e)
))
def on_DbMsr_CreateBackup(self, message):
#TODO: Think how to move the most part of it into Postgresql class
# Retrieve password for scalr pg user
tmpdir = backup_path = None
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
op = operation(name=self._op_backup,
phases=[{
'name': self._phase_backup
}])
op.define()
with op.phase(self._phase_backup):
if not os.path.exists(self._tmp_path):
os.makedirs(self._tmp_path)
# Defining archive name and path
backup_filename = time.strftime(
'%Y-%m-%d-%H:%M:%S') + '.tar.gz'
backup_path = os.path.join(self._tmp_path, backup_filename)
# Creating archive
backup = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=self._tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (
PG_DUMP, db_name, dump_path)
su_args = [
SU_EXEC, '-', self.postgresql.root_user.name, '-c',
pg_args
]
err = system2(su_args)[1]
if err:
raise HandlerError(
'Error while dumping database %s: %s' %
(db_name, err))
backup.add(dump_path, os.path.basename(dump_path))
make_backup_steps(databases, op, _single_backup)
backup.close()
with op.step(self._step_upload_to_cloud_storage):
# Creating list of full paths to archive chunks
if os.path.getsize(
backup_path) > __postgresql__['pgdump_chunk_size']:
parts = [
os.path.join(tmpdir, file) for file in split(
backup_path, backup_filename,
__postgresql__['pgdump_chunk_size'], tmpdir)
]
else:
parts = [backup_path]
sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = self._platform.scalrfs.backups(
BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)",
cloud_storage_path)
trn = transfer.Transfer()
cloud_files = trn.upload(parts, cloud_storage_path)
LOG.info(
"Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(
dict(path=path, size=size)
for path, size in zip(cloud_files, sizes))
op.ok(data=result)
# Notify Scalr
self.send_message(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def do_backup(op):
tmpdir = backup_path = None
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Defining archive name and path
backup_filename = time.strftime('%Y-%m-%d-%H:%M:%S')+'.tar.gz'
backup_path = os.path.join(tmp_path, backup_filename)
# Creating archive
backup_obj = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) #?
backup_obj.add(dump_path, os.path.basename(dump_path))
for db_name in databases:
_single_backup(db_name)
backup_obj.close()
# Creating list of full paths to archive chunks
#if os.path.getsize(backup_path) > __postgresql__['pgdump_chunk_size']:
# parts = [os.path.join(tmpdir, file) for file in split(backup_path, backup_filename, __postgresql__['pgdump_chunk_size'], tmpdir)]
#else:
# parts = [backup_path]
#sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
trn = LargeTransfer(backup_path, cloud_storage_path, tags=backup_tags)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(dict(path=os.path.join(os.path.dirname(manifest.cloudfs_path), c[0]), size=c[2]) for c in
manifest['files'][0]['chunks'])
# Notify Scalr
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result #?
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (
PG_DUMP, db_name, dump_path)
su_args = [
SU_EXEC, '-', self.postgresql.root_user.name, '-c',
pg_args
]
err = system2(su_args)[1]
if err:
raise HandlerError(
'Error while dumping database %s: %s' %
(db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(
BEHAVIOUR)
suffix = 'master' if int(
__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" %
(cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps,
cloud_storage_path,
progress_cb=progress_cb)
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info(
"Postgresql backup uploaded to cloud storage under %s",
cloud_storage_path)
LOG.debug(manifest.data)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))