예제 #1
0
def test__list_files_with_pattern():
    s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key'))
    s3.create_bucket()

    s3.s3_client.put_object(Body='hello world',
                            Bucket='test-bucket',
                            Key='object_1')
    s3.s3_client.put_object(Body='hello world',
                            Bucket='test-bucket',
                            Key='object/foo/bar')
    s3.s3_client.put_object(Body='hello world',
                            Bucket='test-bucket',
                            Key='object_2')

    files_list = s3.list_files(prefix='', pattern='_2')
    assert files_list == ['s3://test-bucket/object_2']

    files_list = s3.list_files(prefix='', pattern='.*/foo/.*')
    assert files_list == ['s3://test-bucket/object/foo/bar']
예제 #2
0
    def destination(self, backup_source=socket.gethostname()):
        """
        :param backup_source: Hostname of the host where backup is taken from.
        :type backup_source: str
        :return: Backup destination instance
        :rtype: BaseDestination
        """
        try:
            backup_destination = self.__cfg.get("destination",
                                                "backup_destination")
            if backup_destination == "ssh":
                return Ssh(
                    self.ssh.path,
                    hostname=backup_source,
                    ssh_host=self.ssh.host,
                    ssh_port=self.ssh.port,
                    ssh_user=self.ssh.user,
                    ssh_key=self.ssh.key,
                )
            elif backup_destination == "s3":
                return S3(
                    bucket=self.s3.bucket,
                    aws_access_key_id=self.s3.aws_access_key_id,
                    aws_secret_access_key=self.s3.aws_secret_access_key,
                    aws_default_region=self.s3.aws_default_region,
                    hostname=backup_source,
                )
            elif backup_destination == "gcs":
                return GCS(
                    bucket=self.gcs.bucket,
                    gc_credentials_file=self.gcs.gc_credentials_file,
                    gc_encryption_key=self.gcs.gc_encryption_key,
                    hostname=backup_source,
                )

            else:
                raise ConfigurationError("Unsupported destination '%s'" %
                                         backup_destination)
        except NoSectionError as err:
            raise ConfigurationError(
                "%s is missing required section 'destination'" %
                self._config_file) from err
예제 #3
0
def test__s3_find_files_returns_sorted(s3_client, config_content_mysql_only,
                                       tmpdir):
    # cleanup the bucket first
    s3_client.delete_all_objects()

    config = tmpdir.join('twindb-backup.cfg')
    content = config_content_mysql_only.format(
        AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'],
        AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'],
        BUCKET=s3_client.bucket,
        daily_copies=5,
        hourly_copies=2)
    config.write(content)

    cmd = [
        'twindb-backup', '--debug', '--config',
        str(config), 'backup', 'daily'
    ]
    n_runs = 3
    for x in xrange(n_runs):
        assert call(cmd) == 0

    dst = S3(
        s3_client.bucket,
        AWSAuthOptions(os.environ['AWS_ACCESS_KEY_ID'],
                       os.environ['AWS_SECRET_ACCESS_KEY']))
    for x in xrange(10):
        result = dst.find_files(dst.remote_path, 'daily')
        assert len(result) == n_runs
        assert result == sorted(result)
        prefix = "{remote_path}/{hostname}/{run_type}/mysql/mysql-".format(
            remote_path=dst.remote_path,
            hostname=socket.gethostname(),
            run_type='daily')
        files = dst.list_files(prefix)
        assert len(files) == n_runs
        assert files == sorted(files)
예제 #4
0
def s3():
    return S3(bucket='test-bucket',
              aws_access_key_id='access_key',
              aws_secret_access_key='secret_key')
예제 #5
0
def get_destination(config, hostname=socket.gethostname()):
    """
    Read config and return instance of Destination class.

    :param config: Tool configuration.
    :type config: ConfigParser.ConfigParser
    :param hostname: Local hostname.
    :type hostname: str
    :return: Instance of destination class.
    :rtype: BaseDestination
    """
    destination = None
    try:
        destination = config.get('destination', 'backup_destination')
        LOG.debug('Destination in the config %s', destination)
        destination = destination.strip('"\'')
    except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
        LOG.critical("Backup destination must be specified "
                     "in the config file")
        exit(-1)

    if destination == "ssh":
        host = config.get('ssh', 'backup_host')
        try:
            port = int(config.get('ssh', 'port'))
        except ConfigParser.NoOptionError:
            port = 22
        try:
            ssh_key = config.get('ssh', 'ssh_key')
        except ConfigParser.NoOptionError:
            ssh_key = '/root/.ssh/id_rsa'
            LOG.debug(
                'ssh_key is not defined in config. '
                'Will use default %s', ssh_key)
        user = config.get('ssh', 'ssh_user')
        remote_path = config.get('ssh', 'backup_dir')
        return Ssh(
            remote_path,
            hostname=hostname,
            ssh_host=host,
            ssh_port=port,
            ssh_user=user,
            ssh_key=ssh_key,
        )

    elif destination == "s3":
        bucket = config.get('s3', 'BUCKET').strip('"\'')
        access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'')
        secret_access_key = config.get('s3',
                                       'AWS_SECRET_ACCESS_KEY').strip('"\'')
        default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'')

        return S3(bucket,
                  AWSAuthOptions(access_key_id,
                                 secret_access_key,
                                 default_region=default_region),
                  hostname=hostname)

    else:
        LOG.critical('Destination %s is not supported', destination)
        raise OperationError('Unsupported destination')
예제 #6
0
파일: test_s3.py 프로젝트: soft-way/backup
def test_basename():
    dst = S3('bucket', AWSAuthOptions('b', 'c'))
    assert dst.basename('s3://bucket/some_dir/some_file.txt') == \
        'some_dir/some_file.txt'
예제 #7
0
파일: test_s3.py 프로젝트: soft-way/backup
def test__create_bucket_creates_the_bucket():
    s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key'))
    s3.create_bucket()

    assert s3.s3_client.head_bucket(Bucket='test-bucket')