def test__list_files_with_pattern():
s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key'))
s3.create_bucket()
s3.s3_client.put_object(Body='hello world',
Bucket='test-bucket',
Key='object_1')
s3.s3_client.put_object(Body='hello world',
Bucket='test-bucket',
Key='object/foo/bar')
s3.s3_client.put_object(Body='hello world',
Bucket='test-bucket',
Key='object_2')
files_list = s3.list_files(prefix='', pattern='_2')
assert files_list == ['s3://test-bucket/object_2']
files_list = s3.list_files(prefix='', pattern='.*/foo/.*')
assert files_list == ['s3://test-bucket/object/foo/bar']
def destination(self, backup_source=socket.gethostname()):
"""
:param backup_source: Hostname of the host where backup is taken from.
:type backup_source: str
:return: Backup destination instance
:rtype: BaseDestination
"""
try:
backup_destination = self.__cfg.get("destination",
"backup_destination")
if backup_destination == "ssh":
return Ssh(
self.ssh.path,
hostname=backup_source,
ssh_host=self.ssh.host,
ssh_port=self.ssh.port,
ssh_user=self.ssh.user,
ssh_key=self.ssh.key,
)
elif backup_destination == "s3":
return S3(
bucket=self.s3.bucket,
aws_access_key_id=self.s3.aws_access_key_id,
aws_secret_access_key=self.s3.aws_secret_access_key,
aws_default_region=self.s3.aws_default_region,
hostname=backup_source,
)
elif backup_destination == "gcs":
return GCS(
bucket=self.gcs.bucket,
gc_credentials_file=self.gcs.gc_credentials_file,
gc_encryption_key=self.gcs.gc_encryption_key,
hostname=backup_source,
)
else:
raise ConfigurationError("Unsupported destination '%s'" %
backup_destination)
except NoSectionError as err:
raise ConfigurationError(
"%s is missing required section 'destination'" %
self._config_file) from err
def test__s3_find_files_returns_sorted(s3_client, config_content_mysql_only,
tmpdir):
# cleanup the bucket first
s3_client.delete_all_objects()
config = tmpdir.join('twindb-backup.cfg')
content = config_content_mysql_only.format(
AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'],
AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'],
BUCKET=s3_client.bucket,
daily_copies=5,
hourly_copies=2)
config.write(content)
cmd = [
'twindb-backup', '--debug', '--config',
str(config), 'backup', 'daily'
]
n_runs = 3
for x in xrange(n_runs):
assert call(cmd) == 0
dst = S3(
s3_client.bucket,
AWSAuthOptions(os.environ['AWS_ACCESS_KEY_ID'],
os.environ['AWS_SECRET_ACCESS_KEY']))
for x in xrange(10):
result = dst.find_files(dst.remote_path, 'daily')
assert len(result) == n_runs
assert result == sorted(result)
prefix = "{remote_path}/{hostname}/{run_type}/mysql/mysql-".format(
remote_path=dst.remote_path,
hostname=socket.gethostname(),
run_type='daily')
files = dst.list_files(prefix)
assert len(files) == n_runs
assert files == sorted(files)
def s3():
return S3(bucket='test-bucket',
aws_access_key_id='access_key',
aws_secret_access_key='secret_key')
def get_destination(config, hostname=socket.gethostname()):
"""
Read config and return instance of Destination class.
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param hostname: Local hostname.
:type hostname: str
:return: Instance of destination class.
:rtype: BaseDestination
"""
destination = None
try:
destination = config.get('destination', 'backup_destination')
LOG.debug('Destination in the config %s', destination)
destination = destination.strip('"\'')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
LOG.critical("Backup destination must be specified "
"in the config file")
exit(-1)
if destination == "ssh":
host = config.get('ssh', 'backup_host')
try:
port = int(config.get('ssh', 'port'))
except ConfigParser.NoOptionError:
port = 22
try:
ssh_key = config.get('ssh', 'ssh_key')
except ConfigParser.NoOptionError:
ssh_key = '/root/.ssh/id_rsa'
LOG.debug(
'ssh_key is not defined in config. '
'Will use default %s', ssh_key)
user = config.get('ssh', 'ssh_user')
remote_path = config.get('ssh', 'backup_dir')
return Ssh(
remote_path,
hostname=hostname,
ssh_host=host,
ssh_port=port,
ssh_user=user,
ssh_key=ssh_key,
)
elif destination == "s3":
bucket = config.get('s3', 'BUCKET').strip('"\'')
access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'')
secret_access_key = config.get('s3',
'AWS_SECRET_ACCESS_KEY').strip('"\'')
default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'')
return S3(bucket,
AWSAuthOptions(access_key_id,
secret_access_key,
default_region=default_region),
hostname=hostname)
else:
LOG.critical('Destination %s is not supported', destination)
raise OperationError('Unsupported destination')
def test_basename():
dst = S3('bucket', AWSAuthOptions('b', 'c'))
assert dst.basename('s3://bucket/some_dir/some_file.txt') == \
'some_dir/some_file.txt'
def test__create_bucket_creates_the_bucket():
s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key'))
s3.create_bucket()
assert s3.s3_client.head_bucket(Bucket='test-bucket')