class TransferTestMixin(object):
    trn = None
    rdst = None
    dst = '/tmp/transfer-sandbox'
 
    def setUp(self):
        self.trn = Transfer(pool=2, max_attempts=1)
        if not os.path.exists(self.dst):
            os.makedirs(self.dst)
 
    def tearDown(self):
        self.remove_local_files()
 
    def make_local_files(self, suffix=None):
        # Create some 1Mb files
        files = []
        #for file in ('chocolate', 'caramel', 'fluids'):
        for file in ('chocolate', 'caramel'):
            file = '%s/%s%s' % (self.dst, file, suffix or '')
            system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
            files.append(file)
        return files
 
    def remove_local_files(self):
        system2('rm -rf %s/*' % self.dst, shell=True)
 
    def test_upload(self):
        files = self.make_local_files('.upload')
        rfiles = self.trn.upload(files, self.rdst)
        assert_transfer(self, files, rfiles, self.rdst)
 
    def test_download(self):
        files = self.make_local_files('.download')
        try:
            rfiles = self.native_upload(files)
        finally:
            self.remove_local_files()
 
        files = self.trn.download(rfiles, self.dst)
        assert_transfer(self, rfiles, files, self.dst)
 
    def native_upload(self, files):
        pass
Beispiel #2
0
class TransferTestMixin(object):
    trn = None
    rdst = None
    dst = '/tmp/transfer-sandbox'

    def setUp(self):
        self.trn = Transfer(pool=2, max_attempts=1)
        if not os.path.exists(self.dst):
            os.makedirs(self.dst)

    def tearDown(self):
        self.remove_local_files()

    def make_local_files(self, suffix=None):
        # Create some 1Mb files
        files = []
        #for file in ('chocolate', 'caramel', 'fluids'):
        for file in ('chocolate', 'caramel'):
            file = '%s/%s%s' % (self.dst, file, suffix or '')
            system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
            files.append(file)
        return files

    def remove_local_files(self):
        system2('rm -rf %s/*' % self.dst, shell=True)

    def test_upload(self):
        files = self.make_local_files('.upload')
        rfiles = self.trn.upload(files, self.rdst)
        assert_transfer(self, files, rfiles, self.rdst)

    def test_download(self):
        files = self.make_local_files('.download')
        try:
            rfiles = self.native_upload(files)
        finally:
            self.remove_local_files()

        files = self.trn.download(rfiles, self.dst)
        assert_transfer(self, rfiles, files, self.dst)

    def native_upload(self, files):
        pass
Beispiel #3
0
        self._logger.info('Uploading %s to Swift under %s' %
                          (local_path, remote_path))
        return self._driver.put(
            local_path, os.path.join(remote_path,
                                     os.path.basename(local_path)))

    def get(self, remote_path, local_path):
        self._logger.info('Downloading %s from Swift to %s' %
                          (remote_path, local_path))
        return self._driver.get(remote_path, local_path)

    def list(self, remote_path):
        return self._driver.ls(remote_path)


Transfer.explore_provider(SwiftTransferProvider)

# Logging


class OpenStackCredentialsLoggerFilter(object):

    request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
    response_re = re.compile('(.*)({["\']access.+})(.*)')

    def filter(self, record):
        message = record.getMessage()
        record.args = ()

        if "passwordCredentials" in message:
            record.msg = 'Requested authentication, credentials are hidden'
Beispiel #4
0
						continue
					raise
				except ValueError, e:
					if 'No JSON object could be decoded' in str(e):
						LOG.debug('Rackspace API failed with invalid JSON response. sleeping %s seconds', interval)
						time.sleep(interval)
						continue
					raise
		return request
	
	C.request = request_decorator(C.request)

_patch_cloudservers()


Transfer.explore_provider(CFTransferProvider)

auth_url = 'https://auth.api.rackspacecloud.com/v1.0'

def _credentials(username=None, api_key=None):
	try:
		username = username or os.environ["CLOUD_SERVERS_USERNAME"]
		api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
		return username, api_key
	except KeyError:
		raise PlatformError('Rackspace API credentials not defined')

def new_cloudserver_conn(username=None, api_key=None):
	return CloudServers(*_credentials(username, api_key))

def new_cloudfiles_conn(username=None, api_key=None, **kwargs):
Beispiel #5
0
        TransferProvider.__init__(self)   

    def put(self, local_path, remote_path):
        self._logger.info('Uploading %s to Swift under %s' % (local_path, remote_path))
        return self._driver.put(local_path, os.path.join(remote_path, os.path.basename(local_path)))
    
    def get(self, remote_path, local_path):
        self._logger.info('Downloading %s from Swift to %s' % (remote_path, local_path))
        return self._driver.get(remote_path, local_path)
        
    
    def list(self, remote_path):
        return self._driver.ls(remote_path)


Transfer.explore_provider(SwiftTransferProvider)

# Logging 

class OpenStackCredentialsLoggerFilter(object):

    request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
    response_re = re.compile('(.*)({["\']access.+})(.*)')

    def filter(self, record):
        message = record.getMessage()
        record.args = ()

        if "passwordCredentials" in message:
            record.msg = 'Requested authentication, credentials are hidden'
            return True
Beispiel #6
0
from __future__ import with_statement

import os
import re
import urllib2
import logging

from scalarizr.bus import bus
from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures
from scalarizr.storage.transfer import Transfer
from .storage import S3TransferProvider

import boto
import boto.ec2

Transfer.explore_provider(S3TransferProvider)
"""
Platform configuration options
"""
OPT_ACCOUNT_ID = "account_id"
OPT_KEY = "key"
OPT_KEY_ID = "key_id"
OPT_EC2_CERT_PATH = "ec2_cert_path"
OPT_CERT_PATH = "cert_path"
OPT_PK_PATH = "pk_path"
"""
User data options
"""
UD_OPT_S3_BUCKET_NAME = "s3bucket"

Beispiel #7
0
from scalarizr.bus import bus
from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures
from scalarizr.storage.transfer import Transfer
from .storage import S3TransferProvider

from boto import connect_s3
from boto.ec2.connection import EC2Connection
from boto.ec2.regioninfo import RegionInfo
import urllib2, re, os


Transfer.explore_provider(S3TransferProvider)


"""
Platform configuration options
"""
OPT_ACCOUNT_ID = "account_id"
OPT_KEY = "key"
OPT_KEY_ID = "key_id"
OPT_EC2_CERT_PATH = "ec2_cert_path"
OPT_CERT_PATH = "cert_path"
OPT_PK_PATH = "pk_path"


"""
User data options 
"""
UD_OPT_S3_BUCKET_NAME = "s3bucket"
 def setUp(self):
     self.trn = Transfer(pool=2, max_attempts=1)
     if not os.path.exists(self.dst):
         os.makedirs(self.dst)
Beispiel #9
0
try:
    import json
except ImportError:
    import simplejson as json

from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build

from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util


Transfer.explore_provider(GoogleCSTransferProvider)

COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'


LOG = logging.getLogger(__name__)


class GoogleApiClientLoggerFilter:
    def filter(self, record):
        if 'takes exactly' or 'takes at most' in record.message:
            return False
        return True

api_logger = logging.getLogger('oauth2client.util')
Beispiel #10
0
                    if 'No JSON object could be decoded' in str(e):
                        LOG.debug(
                            'Rackspace API failed with invalid JSON response. sleeping %s seconds',
                            interval)
                        time.sleep(interval)
                        continue
                    raise

        return request

    C.request = request_decorator(C.request)


_patch_cloudservers()

Transfer.explore_provider(CFTransferProvider)

auth_url = 'https://auth.api.rackspacecloud.com/v1.0'


def _credentials(username=None, api_key=None):
    try:
        username = username or os.environ["CLOUD_SERVERS_USERNAME"]
        api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
        return username, api_key
    except KeyError:
        raise PlatformError('Rackspace API credentials not defined')


def new_cloudserver_conn(username=None, api_key=None):
    return CloudServers(*_credentials(username, api_key))
Beispiel #11
0
 def setUp(self):
     self.trn = Transfer(pool=2, max_attempts=1)
     if not os.path.exists(self.dst):
         os.makedirs(self.dst)
Beispiel #12
0
import threading

try:
    import json
except ImportError:
    import simplejson as json

from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build

from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util

Transfer.explore_provider(GoogleCSTransferProvider)

COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'

LOG = logging.getLogger(__name__)


class GoogleApiClientLoggerFilter:
    def filter(self, record):
        if 'takes exactly' or 'takes at most' in record.message:
            return False
        return True


api_logger = logging.getLogger('oauth2client.util')