def test__list_files_with_pattern(): s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key')) s3.create_bucket() s3.s3_client.put_object(Body='hello world', Bucket='test-bucket', Key='object_1') s3.s3_client.put_object(Body='hello world', Bucket='test-bucket', Key='object/foo/bar') s3.s3_client.put_object(Body='hello world', Bucket='test-bucket', Key='object_2') files_list = s3.list_files(prefix='', pattern='_2') assert files_list == ['s3://test-bucket/object_2'] files_list = s3.list_files(prefix='', pattern='.*/foo/.*') assert files_list == ['s3://test-bucket/object/foo/bar']
def destination(self, backup_source=socket.gethostname()): """ :param backup_source: Hostname of the host where backup is taken from. :type backup_source: str :return: Backup destination instance :rtype: BaseDestination """ try: backup_destination = self.__cfg.get("destination", "backup_destination") if backup_destination == "ssh": return Ssh( self.ssh.path, hostname=backup_source, ssh_host=self.ssh.host, ssh_port=self.ssh.port, ssh_user=self.ssh.user, ssh_key=self.ssh.key, ) elif backup_destination == "s3": return S3( bucket=self.s3.bucket, aws_access_key_id=self.s3.aws_access_key_id, aws_secret_access_key=self.s3.aws_secret_access_key, aws_default_region=self.s3.aws_default_region, hostname=backup_source, ) elif backup_destination == "gcs": return GCS( bucket=self.gcs.bucket, gc_credentials_file=self.gcs.gc_credentials_file, gc_encryption_key=self.gcs.gc_encryption_key, hostname=backup_source, ) else: raise ConfigurationError("Unsupported destination '%s'" % backup_destination) except NoSectionError as err: raise ConfigurationError( "%s is missing required section 'destination'" % self._config_file) from err
def test__s3_find_files_returns_sorted(s3_client, config_content_mysql_only, tmpdir): # cleanup the bucket first s3_client.delete_all_objects() config = tmpdir.join('twindb-backup.cfg') content = config_content_mysql_only.format( AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], BUCKET=s3_client.bucket, daily_copies=5, hourly_copies=2) config.write(content) cmd = [ 'twindb-backup', '--debug', '--config', str(config), 'backup', 'daily' ] n_runs = 3 for x in xrange(n_runs): assert call(cmd) == 0 dst = S3( s3_client.bucket, AWSAuthOptions(os.environ['AWS_ACCESS_KEY_ID'], os.environ['AWS_SECRET_ACCESS_KEY'])) for x in xrange(10): result = dst.find_files(dst.remote_path, 'daily') assert len(result) == n_runs assert result == sorted(result) prefix = "{remote_path}/{hostname}/{run_type}/mysql/mysql-".format( remote_path=dst.remote_path, hostname=socket.gethostname(), run_type='daily') files = dst.list_files(prefix) assert len(files) == n_runs assert files == sorted(files)
def s3(): return S3(bucket='test-bucket', aws_access_key_id='access_key', aws_secret_access_key='secret_key')
def get_destination(config, hostname=socket.gethostname()): """ Read config and return instance of Destination class. :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param hostname: Local hostname. :type hostname: str :return: Instance of destination class. :rtype: BaseDestination """ destination = None try: destination = config.get('destination', 'backup_destination') LOG.debug('Destination in the config %s', destination) destination = destination.strip('"\'') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): LOG.critical("Backup destination must be specified " "in the config file") exit(-1) if destination == "ssh": host = config.get('ssh', 'backup_host') try: port = int(config.get('ssh', 'port')) except ConfigParser.NoOptionError: port = 22 try: ssh_key = config.get('ssh', 'ssh_key') except ConfigParser.NoOptionError: ssh_key = '/root/.ssh/id_rsa' LOG.debug( 'ssh_key is not defined in config. ' 'Will use default %s', ssh_key) user = config.get('ssh', 'ssh_user') remote_path = config.get('ssh', 'backup_dir') return Ssh( remote_path, hostname=hostname, ssh_host=host, ssh_port=port, ssh_user=user, ssh_key=ssh_key, ) elif destination == "s3": bucket = config.get('s3', 'BUCKET').strip('"\'') access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'') secret_access_key = config.get('s3', 'AWS_SECRET_ACCESS_KEY').strip('"\'') default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'') return S3(bucket, AWSAuthOptions(access_key_id, secret_access_key, default_region=default_region), hostname=hostname) else: LOG.critical('Destination %s is not supported', destination) raise OperationError('Unsupported destination')
def test_basename(): dst = S3('bucket', AWSAuthOptions('b', 'c')) assert dst.basename('s3://bucket/some_dir/some_file.txt') == \ 'some_dir/some_file.txt'
def test__create_bucket_creates_the_bucket(): s3 = S3('test-bucket', AWSAuthOptions('access_key', 'secret_key')) s3.create_bucket() assert s3.s3_client.head_bucket(Bucket='test-bucket')