Пример #1
0
class TransferTestMixin(object):
    trn = None
    rdst = None
    dst = '/tmp/transfer-sandbox'
 
    def setUp(self):
        self.trn = Transfer(pool=2, max_attempts=1)
        if not os.path.exists(self.dst):
            os.makedirs(self.dst)
 
    def tearDown(self):
        self.remove_local_files()
 
    def make_local_files(self, suffix=None):
        # Create some 1Mb files
        files = []
        #for file in ('chocolate', 'caramel', 'fluids'):
        for file in ('chocolate', 'caramel'):
            file = '%s/%s%s' % (self.dst, file, suffix or '')
            system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
            files.append(file)
        return files
 
    def remove_local_files(self):
        system2('rm -rf %s/*' % self.dst, shell=True)
 
    def test_upload(self):
        files = self.make_local_files('.upload')
        rfiles = self.trn.upload(files, self.rdst)
        assert_transfer(self, files, rfiles, self.rdst)
 
    def test_download(self):
        files = self.make_local_files('.download')
        try:
            rfiles = self.native_upload(files)
        finally:
            self.remove_local_files()
 
        files = self.trn.download(rfiles, self.dst)
        assert_transfer(self, rfiles, files, self.dst)
 
    def native_upload(self, files):
        pass
Пример #2
0
class TransferTestMixin(object):
    trn = None
    rdst = None
    dst = '/tmp/transfer-sandbox'

    def setUp(self):
        self.trn = Transfer(pool=2, max_attempts=1)
        if not os.path.exists(self.dst):
            os.makedirs(self.dst)

    def tearDown(self):
        self.remove_local_files()

    def make_local_files(self, suffix=None):
        # Create some 1Mb files
        files = []
        #for file in ('chocolate', 'caramel', 'fluids'):
        for file in ('chocolate', 'caramel'):
            file = '%s/%s%s' % (self.dst, file, suffix or '')
            system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
            files.append(file)
        return files

    def remove_local_files(self):
        system2('rm -rf %s/*' % self.dst, shell=True)

    def test_upload(self):
        files = self.make_local_files('.upload')
        rfiles = self.trn.upload(files, self.rdst)
        assert_transfer(self, files, rfiles, self.rdst)

    def test_download(self):
        files = self.make_local_files('.download')
        try:
            rfiles = self.native_upload(files)
        finally:
            self.remove_local_files()

        files = self.trn.download(rfiles, self.dst)
        assert_transfer(self, rfiles, files, self.dst)

    def native_upload(self, files):
        pass
Пример #3
0
        self._logger.info('Uploading %s to Swift under %s' %
                          (local_path, remote_path))
        return self._driver.put(
            local_path, os.path.join(remote_path,
                                     os.path.basename(local_path)))

    def get(self, remote_path, local_path):
        self._logger.info('Downloading %s from Swift to %s' %
                          (remote_path, local_path))
        return self._driver.get(remote_path, local_path)

    def list(self, remote_path):
        return self._driver.ls(remote_path)


Transfer.explore_provider(SwiftTransferProvider)

# Logging


class OpenStackCredentialsLoggerFilter(object):

    request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
    response_re = re.compile('(.*)({["\']access.+})(.*)')

    def filter(self, record):
        message = record.getMessage()
        record.args = ()

        if "passwordCredentials" in message:
            record.msg = 'Requested authentication, credentials are hidden'
Пример #4
0
						continue
					raise
				except ValueError, e:
					if 'No JSON object could be decoded' in str(e):
						LOG.debug('Rackspace API failed with invalid JSON response. sleeping %s seconds', interval)
						time.sleep(interval)
						continue
					raise
		return request
	
	C.request = request_decorator(C.request)

_patch_cloudservers()


Transfer.explore_provider(CFTransferProvider)

auth_url = 'https://auth.api.rackspacecloud.com/v1.0'

def _credentials(username=None, api_key=None):
	try:
		username = username or os.environ["CLOUD_SERVERS_USERNAME"]
		api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
		return username, api_key
	except KeyError:
		raise PlatformError('Rackspace API credentials not defined')

def new_cloudserver_conn(username=None, api_key=None):
	return CloudServers(*_credentials(username, api_key))

def new_cloudfiles_conn(username=None, api_key=None, **kwargs):
Пример #5
0
        TransferProvider.__init__(self)   

    def put(self, local_path, remote_path):
        self._logger.info('Uploading %s to Swift under %s' % (local_path, remote_path))
        return self._driver.put(local_path, os.path.join(remote_path, os.path.basename(local_path)))
    
    def get(self, remote_path, local_path):
        self._logger.info('Downloading %s from Swift to %s' % (remote_path, local_path))
        return self._driver.get(remote_path, local_path)
        
    
    def list(self, remote_path):
        return self._driver.ls(remote_path)


Transfer.explore_provider(SwiftTransferProvider)

# Logging 

class OpenStackCredentialsLoggerFilter(object):

    request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
    response_re = re.compile('(.*)({["\']access.+})(.*)')

    def filter(self, record):
        message = record.getMessage()
        record.args = ()

        if "passwordCredentials" in message:
            record.msg = 'Requested authentication, credentials are hidden'
            return True
Пример #6
0
from __future__ import with_statement

import os
import re
import urllib2
import logging

from scalarizr.bus import bus
from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures
from scalarizr.storage.transfer import Transfer
from .storage import S3TransferProvider

import boto
import boto.ec2

Transfer.explore_provider(S3TransferProvider)
"""
Platform configuration options
"""
OPT_ACCOUNT_ID = "account_id"
OPT_KEY = "key"
OPT_KEY_ID = "key_id"
OPT_EC2_CERT_PATH = "ec2_cert_path"
OPT_CERT_PATH = "cert_path"
OPT_PK_PATH = "pk_path"
"""
User data options
"""
UD_OPT_S3_BUCKET_NAME = "s3bucket"

Пример #7
0
from scalarizr.bus import bus
from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures
from scalarizr.storage.transfer import Transfer
from .storage import S3TransferProvider

from boto import connect_s3
from boto.ec2.connection import EC2Connection
from boto.ec2.regioninfo import RegionInfo
import urllib2, re, os


Transfer.explore_provider(S3TransferProvider)


"""
Platform configuration options
"""
OPT_ACCOUNT_ID = "account_id"
OPT_KEY = "key"
OPT_KEY_ID = "key_id"
OPT_EC2_CERT_PATH = "ec2_cert_path"
OPT_CERT_PATH = "cert_path"
OPT_PK_PATH = "pk_path"


"""
User data options 
"""
UD_OPT_S3_BUCKET_NAME = "s3bucket"
Пример #8
0
 def setUp(self):
     self.trn = Transfer(pool=2, max_attempts=1)
     if not os.path.exists(self.dst):
         os.makedirs(self.dst)
Пример #9
0
try:
    import json
except ImportError:
    import simplejson as json

from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build

from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util


Transfer.explore_provider(GoogleCSTransferProvider)

COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'


LOG = logging.getLogger(__name__)


class GoogleApiClientLoggerFilter:
    def filter(self, record):
        if 'takes exactly' or 'takes at most' in record.message:
            return False
        return True

api_logger = logging.getLogger('oauth2client.util')
Пример #10
0
                    if 'No JSON object could be decoded' in str(e):
                        LOG.debug(
                            'Rackspace API failed with invalid JSON response. sleeping %s seconds',
                            interval)
                        time.sleep(interval)
                        continue
                    raise

        return request

    C.request = request_decorator(C.request)


_patch_cloudservers()

Transfer.explore_provider(CFTransferProvider)

auth_url = 'https://auth.api.rackspacecloud.com/v1.0'


def _credentials(username=None, api_key=None):
    try:
        username = username or os.environ["CLOUD_SERVERS_USERNAME"]
        api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
        return username, api_key
    except KeyError:
        raise PlatformError('Rackspace API credentials not defined')


def new_cloudserver_conn(username=None, api_key=None):
    return CloudServers(*_credentials(username, api_key))
Пример #11
0
 def setUp(self):
     self.trn = Transfer(pool=2, max_attempts=1)
     if not os.path.exists(self.dst):
         os.makedirs(self.dst)
Пример #12
0
import threading

try:
    import json
except ImportError:
    import simplejson as json

from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build

from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util

Transfer.explore_provider(GoogleCSTransferProvider)

COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'

LOG = logging.getLogger(__name__)


class GoogleApiClientLoggerFilter:
    def filter(self, record):
        if 'takes exactly' or 'takes at most' in record.message:
            return False
        return True


api_logger = logging.getLogger('oauth2client.util')