class TransferTestMixin(object):
trn = None
rdst = None
dst = '/tmp/transfer-sandbox'
def setUp(self):
self.trn = Transfer(pool=2, max_attempts=1)
if not os.path.exists(self.dst):
os.makedirs(self.dst)
def tearDown(self):
self.remove_local_files()
def make_local_files(self, suffix=None):
# Create some 1Mb files
files = []
#for file in ('chocolate', 'caramel', 'fluids'):
for file in ('chocolate', 'caramel'):
file = '%s/%s%s' % (self.dst, file, suffix or '')
system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
files.append(file)
return files
def remove_local_files(self):
system2('rm -rf %s/*' % self.dst, shell=True)
def test_upload(self):
files = self.make_local_files('.upload')
rfiles = self.trn.upload(files, self.rdst)
assert_transfer(self, files, rfiles, self.rdst)
def test_download(self):
files = self.make_local_files('.download')
try:
rfiles = self.native_upload(files)
finally:
self.remove_local_files()
files = self.trn.download(rfiles, self.dst)
assert_transfer(self, rfiles, files, self.dst)
def native_upload(self, files):
pass
class TransferTestMixin(object):
trn = None
rdst = None
dst = '/tmp/transfer-sandbox'
def setUp(self):
self.trn = Transfer(pool=2, max_attempts=1)
if not os.path.exists(self.dst):
os.makedirs(self.dst)
def tearDown(self):
self.remove_local_files()
def make_local_files(self, suffix=None):
# Create some 1Mb files
files = []
#for file in ('chocolate', 'caramel', 'fluids'):
for file in ('chocolate', 'caramel'):
file = '%s/%s%s' % (self.dst, file, suffix or '')
system2('dd if=/dev/urandom of=%s bs=100K count=1' % file, shell=True)
files.append(file)
return files
def remove_local_files(self):
system2('rm -rf %s/*' % self.dst, shell=True)
def test_upload(self):
files = self.make_local_files('.upload')
rfiles = self.trn.upload(files, self.rdst)
assert_transfer(self, files, rfiles, self.rdst)
def test_download(self):
files = self.make_local_files('.download')
try:
rfiles = self.native_upload(files)
finally:
self.remove_local_files()
files = self.trn.download(rfiles, self.dst)
assert_transfer(self, rfiles, files, self.dst)
def native_upload(self, files):
pass
self._logger.info('Uploading %s to Swift under %s' %
(local_path, remote_path))
return self._driver.put(
local_path, os.path.join(remote_path,
os.path.basename(local_path)))
def get(self, remote_path, local_path):
self._logger.info('Downloading %s from Swift to %s' %
(remote_path, local_path))
return self._driver.get(remote_path, local_path)
def list(self, remote_path):
return self._driver.ls(remote_path)
Transfer.explore_provider(SwiftTransferProvider)
# Logging
class OpenStackCredentialsLoggerFilter(object):
request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
response_re = re.compile('(.*)({["\']access.+})(.*)')
def filter(self, record):
message = record.getMessage()
record.args = ()
if "passwordCredentials" in message:
record.msg = 'Requested authentication, credentials are hidden'
continue
raise
except ValueError, e:
if 'No JSON object could be decoded' in str(e):
LOG.debug('Rackspace API failed with invalid JSON response. sleeping %s seconds', interval)
time.sleep(interval)
continue
raise
return request
C.request = request_decorator(C.request)
_patch_cloudservers()
Transfer.explore_provider(CFTransferProvider)
auth_url = 'https://auth.api.rackspacecloud.com/v1.0'
def _credentials(username=None, api_key=None):
try:
username = username or os.environ["CLOUD_SERVERS_USERNAME"]
api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
return username, api_key
except KeyError:
raise PlatformError('Rackspace API credentials not defined')
def new_cloudserver_conn(username=None, api_key=None):
return CloudServers(*_credentials(username, api_key))
def new_cloudfiles_conn(username=None, api_key=None, **kwargs):
TransferProvider.__init__(self)
def put(self, local_path, remote_path):
self._logger.info('Uploading %s to Swift under %s' % (local_path, remote_path))
return self._driver.put(local_path, os.path.join(remote_path, os.path.basename(local_path)))
def get(self, remote_path, local_path):
self._logger.info('Downloading %s from Swift to %s' % (remote_path, local_path))
return self._driver.get(remote_path, local_path)
def list(self, remote_path):
return self._driver.ls(remote_path)
Transfer.explore_provider(SwiftTransferProvider)
# Logging
class OpenStackCredentialsLoggerFilter(object):
request_re = re.compile('(X-Auth[^:]+:)([^\'"])+')
response_re = re.compile('(.*)({["\']access.+})(.*)')
def filter(self, record):
message = record.getMessage()
record.args = ()
if "passwordCredentials" in message:
record.msg = 'Requested authentication, credentials are hidden'
return True
from __future__ import with_statement import os import re import urllib2 import logging from scalarizr.bus import bus from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures from scalarizr.storage.transfer import Transfer from .storage import S3TransferProvider import boto import boto.ec2 Transfer.explore_provider(S3TransferProvider) """ Platform configuration options """ OPT_ACCOUNT_ID = "account_id" OPT_KEY = "key" OPT_KEY_ID = "key_id" OPT_EC2_CERT_PATH = "ec2_cert_path" OPT_CERT_PATH = "cert_path" OPT_PK_PATH = "pk_path" """ User data options """ UD_OPT_S3_BUCKET_NAME = "s3bucket"
from scalarizr.bus import bus from scalarizr.platform import Ec2LikePlatform, PlatformError, PlatformFeatures from scalarizr.storage.transfer import Transfer from .storage import S3TransferProvider from boto import connect_s3 from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo import urllib2, re, os Transfer.explore_provider(S3TransferProvider) """ Platform configuration options """ OPT_ACCOUNT_ID = "account_id" OPT_KEY = "key" OPT_KEY_ID = "key_id" OPT_EC2_CERT_PATH = "ec2_cert_path" OPT_CERT_PATH = "cert_path" OPT_PK_PATH = "pk_path" """ User data options """ UD_OPT_S3_BUCKET_NAME = "s3bucket"
def setUp(self):
self.trn = Transfer(pool=2, max_attempts=1)
if not os.path.exists(self.dst):
os.makedirs(self.dst)
try:
import json
except ImportError:
import simplejson as json
from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build
from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util
Transfer.explore_provider(GoogleCSTransferProvider)
COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'
LOG = logging.getLogger(__name__)
class GoogleApiClientLoggerFilter:
def filter(self, record):
if 'takes exactly' or 'takes at most' in record.message:
return False
return True
api_logger = logging.getLogger('oauth2client.util')
if 'No JSON object could be decoded' in str(e):
LOG.debug(
'Rackspace API failed with invalid JSON response. sleeping %s seconds',
interval)
time.sleep(interval)
continue
raise
return request
C.request = request_decorator(C.request)
_patch_cloudservers()
Transfer.explore_provider(CFTransferProvider)
auth_url = 'https://auth.api.rackspacecloud.com/v1.0'
def _credentials(username=None, api_key=None):
try:
username = username or os.environ["CLOUD_SERVERS_USERNAME"]
api_key = api_key or os.environ['CLOUD_SERVERS_API_KEY']
return username, api_key
except KeyError:
raise PlatformError('Rackspace API credentials not defined')
def new_cloudserver_conn(username=None, api_key=None):
return CloudServers(*_credentials(username, api_key))
def setUp(self):
self.trn = Transfer(pool=2, max_attempts=1)
if not os.path.exists(self.dst):
os.makedirs(self.dst)
import threading
try:
import json
except ImportError:
import simplejson as json
from oauth2client.client import SignedJwtAssertionCredentials
from apiclient.discovery import build
from scalarizr.platform import Platform
from scalarizr.storage.transfer import Transfer
from scalarizr.platform.gce.storage import GoogleCSTransferProvider
from scalarizr import util
Transfer.explore_provider(GoogleCSTransferProvider)
COMPUTE_RW_SCOPE = 'https://www.googleapis.com/auth/compute'
STORAGE_FULL_SCOPE = 'https://www.googleapis.com/auth/devstorage.full_control'
LOG = logging.getLogger(__name__)
class GoogleApiClientLoggerFilter:
def filter(self, record):
if 'takes exactly' or 'takes at most' in record.message:
return False
return True
api_logger = logging.getLogger('oauth2client.util')