Пример #1
0
def put_s3(i, args):
    print 'Thread %4d: Start' % i
    cf_driver = get_driver(Provider.CLOUDFILES_US)
    cf = cf_driver(args.cf_username, args.cf_password,
                   ex_force_service_region=args.cf_region)
    cf_cont = cf.get_container(args.cf_container)

    s3_driver = get_driver(Provider.S3)
    s3 = s3_driver(args.s3_access_id, args.s3_access_key)
    s3_cont = s3.get_container(args.s3_container)

    while 1:
        try:
            obj = queue.get_nowait()
        except gevent.queue.Empty:
            print 'Thread %4d: Queue empty' % i
            raise gevent.GreenletExit
        else:
            obj.driver = cf
            obj.container = cf_cont

            print 'Thread %4d: Upload %s' % (i, obj.name)
            obj_stream = obj.as_stream()
            s3_cont.upload_object_via_stream(obj_stream, obj.name,
                                             extra=obj.extra)
            print 'Thread %4d: Upload complete %s' % (i, obj.name)
    print 'Thread %4d: Complete' % i
def get_driver_helper(provider_name):
    if provider_name == 'cloudfiles':
        return get_driver(Provider.CLOUDFILES_US)
    elif provider_name == 's3':
        return get_driver(Provider.S3)
    else:
        print 'Unknown provider %s!' % provider_name
        sys.exit(1)
Пример #3
0
def html(revision_id, path="index.html"):
    container = Container(
        app.config["BUCKET"],
        None,
        get_driver(Provider.CLOUDFILES)(
            app.config["RACKSPACE_USER"],
            app.config["RACKSPACE_APIKEY"],
            region=app.config["RACKSPACE_REGION"],
        ),
    )

    # See if the requested file exists
    try:
        obj = container.get_object(
            "html/{revision}/{path}".format(revision=revision_id, path=path)
        )
    except ObjectDoesNotExistError:
        abort(404)

    # Get the requested file
    data = b""
    for chunk in obj.as_stream():
        data += chunk

    return data, 200, {k.replace("_", "-"): v for k, v in obj.extra.items()}
Пример #4
0
    def __init__(self, config=None):
        self.provider = config
        extra_kwargs = {}
        if 'region' in self.provider:
            extra_kwargs['region'] = self.provider['region']
        try:
            provider_type = self.provider['type']
            if isinstance(provider_type, str):
                module_path, tag = provider_type.rsplit('.', 1)
                if module_path != 'libcloud.storage.types.Provider':
                    raise ValueError("Invalid module path")
                provider_type = getattr(Provider, tag)

            Driver = get_driver(provider_type)
            self.driver = Driver(
                self.provider['user'],
                self.provider['key'],
                **extra_kwargs
            )
        except Exception as e:
            raise ImproperlyConfigured(
                "Unable to create libcloud driver type %s: %s" %
                (self.provider.get('type'), e))
        self.bucket = self.provider['bucket']   # Limit to one container
        self.base_url = self.provider['base_url']
 def __init__(self, provider_name, option=None):
     self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
     if not self.provider:
         raise ImproperlyConfigured(
             'LIBCLOUD_PROVIDERS %s not define or invalid' % provider_name)
     try:
         Driver = get_driver(self.provider['type'])
         
         user = self.provide.get('user', None)
         key = self.provider.get('key', None)
         
         if provider_name == 'google' and not user:
             self.driver = Driver(
                 key,
                 secret=self.provider['secret'],
                 project=self.provider['project']
                 )
         else:
             self.driver = Driver(
                 user,
                 key,
                 )
     except Exception, e:
         raise ImproperlyConfigured(
             "Unable to create libcloud driver type %s" % \
             (self.provider.get('type'), e))
    def __init__(self, provider_name=None, option=None):
        if provider_name is None:
            provider_name = getattr(settings, 'DEFAULT_LIBCLOUD_PROVIDER', 'default')

        self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
        if not self.provider:
            raise ImproperlyConfigured(
                'LIBCLOUD_PROVIDERS %s not defined or invalid' % provider_name)
        extra_kwargs = {}
        if 'region' in self.provider:
            extra_kwargs['region'] = self.provider['region']
        try:
            provider_type = self.provider['type']
            if isinstance(provider_type, basestring):
                module_path, tag = provider_type.rsplit('.', 1)
                if module_path != 'libcloud.storage.types.Provider':
                    raise ValueError("Invalid module path")
                provider_type = getattr(Provider, tag)

            Driver = get_driver(provider_type)
            self.driver = Driver(
                self.provider['user'],
                self.provider['key'],
                **extra_kwargs
            )
        except Exception as e:
            raise ImproperlyConfigured(
                "Unable to create libcloud driver type %s: %s" % \
                (self.provider.get('type'), e))
        self.bucket = self.provider['bucket']   # Limit to one container
Пример #7
0
 def _get_storage(uri):
     """
     Given a URI like local:///srv/repo or s3://key:secret/apt.example.com,
     return a libcloud storage or container object.
     """
     driver = get_driver(uri.scheme)
     key = uri.username
     secret = uri.password
     container = uri.netloc
     if uri.scheme.startswith('s3'):
         if not key:
             key = os.environ.get('AWS_ACCESS_KEY_ID')
         if not secret:
             secret = os.environ.get('AWS_SECRET_ACCESS_KEY')
         if not (key and secret and container):
             raise ValueError('For S3 you must provide an access key ID, secret access key, and bucket name')
     elif uri.scheme == 'local':
         parts = []
         if uri.netloc:
             parts.append(uri.netloc)
         if uri.path:
             parts.append(uri.path)
         if not parts:
             parts.append('.')
         base_path = os.path.abspath(''.join(parts))
         key = os.path.dirname(base_path)
         container = os.path.basename(base_path)
     storage = driver(key, secret)
     try:
         return storage.get_container(container)
     except ContainerDoesNotExistError:
         return storage.create_container(container)
Пример #8
0
    def handle_noargs(self, **options):
        try:
            cls = get_driver(Provider.AZURE_BLOBS)
            driver = cls(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)
            container = driver.get_container(container_name=settings.AZURE_CONTAINER)

            # .filter(upload_date__lte=datetime.today() - timedelta(days=180)) \
            mixes = (
                Mix.objects.exclude(archive_path__isnull=False)
                .annotate(num_plays=Count("activity_plays"))
                .order_by("num_plays")
            )
            for mix in mixes:
                if os.path.isfile(mix.get_absolute_path()):
                    print "Uploading file for: %s" % mix.slug
                    file_name = "%s.%s" % (mix.uid, mix.filetype)
                    archive_path = url_path_join(settings.AZURE_ITEM_BASE_URL, settings.AZURE_CONTAINER, file_name)

                    with open(mix.get_absolute_path(), "rb") as iterator:
                        obj = driver.upload_object_via_stream(
                            iterator=iterator, container=container, object_name=file_name
                        )
                        print "Uploaded"
                        mix.archive_path = archive_path
                        mix.save()

                        expired_path = join(settings.MEDIA_ROOT, "mixes/archived")
                        new_file = os.path.join(expired_path, basename(iterator.name))
                        os.rename(iterator.name, new_file)

                    print "done- file is %s" % mix.archive_path

        except Exception, ex:
            print "Debug exception: %s" % ex.message
Пример #9
0
    def driver(self):
        try:
            cloud_driver = get_driver(self._cloud_name)
            param_dict = self.get_driver_params()
        except:
            raise ValueError

        return cloud_driver(**param_dict)
Пример #10
0
 def __init__(self):
     self.driver = get_driver(
         getattr(
             Provider,
             self.driver_name
         )
     )(**self.driver_options)
     self.container = self.driver.get_container(
         container_name=self.container_name
     )
Пример #11
0
    def cloudy(self):
        S3_ACCESS_ID = os.getenv('AWS_ACCESS_KEY_ID')
        S3_SECRET_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')

        Driver = get_driver(Provider.S3)
        conn = Driver(S3_ACCESS_ID, S3_SECRET_KEY)

        containers = conn.list_containers()
        print('%d containers:' % len(containers))
        for container in containers:
            print(' - %s' % container.name)
Пример #12
0
    def setup(self):
        print "mounting test filesystem"
        args = shlex.split(self.MOUNT_CMD)
        p = subprocess.Popen(args)
        p.wait()

        if 0 != p.returncode:
            print >>sys.stderr, "failed to mount filesystem"
            sys.exit(1)

        print "setting up libcloud storage connection"

        self.storage_handle = get_driver(getattr(Provider, driver))(access_id, secret)
        print self.storage_handle
Пример #13
0
def create_destination(args):
    cf_driver = get_driver(Provider.CLOUDFILES_US)
    cf = cf_driver(args.cf_username, args.cf_password,
                   ex_force_service_region=args.cf_region)
    s3_driver = get_driver(Provider.S3)
    s3 = s3_driver(args.s3_access_id, args.s3_access_key)

    if args.from_service == 'S3':
        try:
            cf.create_container(args.cf_container)
        except libcloud.storage.types.ContainerAlreadyExistsError:
            pass
        except Exception as e:
            raise SystemExit(e)
        return s3.get_container(args.s3_container)
    elif args.from_service == 'CF':
        try:
            s3.create_container(args.s3_container)
        except (libcloud.storage.types.ContainerAlreadyExistsError):
            pass
        except Exception as e:
            raise SystemExit(e)
        return cf.get_container(args.cf_container)
Пример #14
0
 def __init__(self, provider_name, option=None):
     self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
     if not self.provider:
         raise ImproperlyConfigured(
             'LIBCLOUD_PROVIDERS %s not define or invalid' % provider_name)
     try:
         Driver = get_driver(self.provider['type'])
         self.driver = Driver(
             self.provider['user'],
             self.provider['key'],
             )
     except Exception, e:
         raise ImproperlyConfigured(
             "Unable to create libcloud driver type %s" % \
             (self.provider.get('type'), e))
Пример #15
0
def get_driver_class(provider):
    """
    Return the driver class
    :param provider: str - provider name
    :return:
    """
    if "." in provider:
        parts = provider.split('.')
        kls = parts.pop()
        path = '.'.join(parts)
        module = import_module(path)
        if not hasattr(module, kls):
            raise ImportError('{0} provider not found at {1}'.format(
                kls,
                path))
        driver = getattr(module, kls)
    else:
        driver = getattr(Provider, provider.upper())
    return get_driver(driver)
Пример #16
0
    def __init__(self, provider_name=None, option=None):
        if provider_name is None:
            provider_name = getattr(settings, "DEFAULT_LIBCLOUD_PROVIDER", "default")

        self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
        if not self.provider:
            raise ImproperlyConfigured("LIBCLOUD_PROVIDERS %s not defined or invalid" % provider_name)
        try:
            provider_type = self.provider["type"]
            if isinstance(provider_type, basestring):
                module_path, tag = provider_type.rsplit(".", 1)
                if module_path != "libcloud.storage.types.Provider":
                    raise ValueError("Invalid module path")
                provider_type = getattr(Provider, tag)

            Driver = get_driver(provider_type)
            self.driver = Driver(self.provider["user"], self.provider["key"])
        except Exception, e:
            raise ImproperlyConfigured("Unable to create libcloud driver type %s: %s" % (self.provider.get("type"), e))
Пример #17
0
def test_cloud_master_key_store_s3(request, tmpdir):
    try:
        access_key = request.config.getoption('--aws-access-key')
        secret_key = request.config.getoption('--aws-secret-key')
        bucket_name = request.config.getoption('--aws-s3-bucket')
    except ValueError:
        access_key = secret_key = bucket_name = None
    if access_key is None or secret_key is None or bucket_name is None:
        skip(
            '--aws-access-key/--aws-secret-key/--aws-s3-bucket are not '
            'provided; skipped'
        )
    driver_cls = get_driver(Provider.S3)
    driver = driver_cls(access_key, secret_key)
    container = driver.get_container(container_name=bucket_name)
    tmpname = ''.join(map('{:02x}'.format, os.urandom(16)))
    s = CloudMasterKeyStore(driver, container, tmpname)
    key = RSAKey.generate(1024)
    # load() -- when not exists
    with raises(EmptyStoreError):
        s.load()
    try:
        # save()
        s.save(key)
        obj = driver.get_object(container.name, tmpname)
        dest = tmpdir / tmpname
        obj.download(str(dest))
        saved = read_private_key_file(dest.open())
        assert isinstance(saved, RSAKey)
        assert saved.get_base64() == key.get_base64()
        # load() -- when exists
        loaded = s.load()
        assert isinstance(loaded, RSAKey)
        assert loaded.get_base64() == key.get_base64()
    finally:
        try:
            o = driver.get_object(container.name, tmpname)
        except ObjectDoesNotExistError:
            pass
        else:
            o.delete()
Пример #18
0
    def backup(self, dest_provider, dest_type, host):
        import subprocess
        from datetime import datetime

        from libcloud.storage.types import Provider, ContainerDoesNotExistError
        from libcloud.storage.providers import get_driver
        import libcloud.security

        # TODO Make this optional
        libcloud.security.VERIFY_SSL_CERT = False

        print host

        driver = get_driver(getattr(Provider, dest_provider))(self.config.destination_key,
                                  self.config.destination_secret)

        directory = expand_paths([self.config.source_name])
        cmd = 'tar cvzpf - %s' % (' '.join(directory))

        object_name = '%s-%s.tar.gz' % (self.config.destination_prefix, datetime.now().strftime('%Y-%m-%d'))
        container_name = self.config.destination_container

        # Create a container if it doesn't already exist
        try:
            container = driver.get_container(container_name=container_name)
        except ContainerDoesNotExistError:
            container = driver.create_container(container_name=container_name)

        pipe = subprocess.Popen(cmd, bufsize=0, shell=True, stdout=subprocess.PIPE)
        return_code = pipe.poll()

        print 'Uploading object...'

        while return_code is None:
            # Compress data in our directory and stream it directly to CF
            obj = container.upload_object_via_stream(iterator=pipe.stdout,
                                                    object_name=object_name)
            return_code = pipe.poll()

        print 'Upload complete, transferred: %s KB' % ((obj.size / 1024))
Пример #19
0
    def __init__(self, **options):
        """ Build a Cloud Provider. """
        super(DefaultCloudProvider, self).__init__(**options)
        try:
            cls = get_driver(self._name)
            self._driver = cls(options['id'], options['key'])

            self._container = self._driver.get_container(self._container_name)
            logger.info(
                "Connected to %s container from %s provider.",
                self._container_name, self._name)

        except AttributeError:
            msg = "Provider %s doesn't exists in detours."
            self._handle_error(BadProviderError, msg, self._name)

        except ContainerDoesNotExistError:
            msg = "Container %s does not exists in %s provider."
            self._handle_error(
                BadContainerError, msg, self._container_name, self._name)

        except Exception:
            msg = "Fatal error while creating %s provider."
            self._handle_error(CloudDetoursError, msg, self._name)
Пример #20
0
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

account_id = 'XXXXXX'
application_key = 'YYYYYY'

cls = get_driver(Provider.BACKBLAZE_B2)
driver = cls(account_id, application_key)
#!/usr/bin/env python
# Code samples for blog post
# http://mikethetechie.com/post/6975966936/controlling-the-environment-cloud-control-apis
#
# Note - I couldn't get this to work. For more info, see http://mikethetechie.com/private/6969971577/tumblr_lng0mzd5bg1qf6p57
import secrets # import user name RACKSPACE_USER and API key RACKSPACE_KEY

from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
from pprint import pprint




Driver = get_driver(Provider.CLOUDFILES_UK)
conn2 = Driver(secrets.RACKSPACE_USER, secrets.RACKSPACE_KEY, 'https://lon.auth.api.rackspacecloud.com/v1.0')


containers = conn2.list_containers()
container_objects = conn2.list_container_objects(containers[0])

pprint(containers)
pprint(container_objects)


cont = conn2.create_container('Testlibcloudcontainer')
obj = cont.create_object('t.html')
obj.content_type = 'text/html'

#Finally for some data.
obj.write('<html><head></head><body><img src="xd_logo.jpg"><br />Hello world!</body></html>')
Пример #22
0
 def _azure_client(self) -> Container:
     driver = get_driver(self._provider)
     client = driver(self._account, self._key)
     return client.get_container(self._container)
Пример #23
0
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pprint import pprint

from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

CloudFiles = get_driver(Provider.CLOUDFILES_UK)

driver = CloudFiles('access key id', 'secret key')

containers = driver.list_containers()
container_objects = driver.list_container_objects(containers[0])

pprint(containers)
pprint(container_objects)
Пример #24
0
    rootstring=''
    datestamp=''
    container_name = ('')
# Set up logging file
logfilename = loggydatestamp + '-Shoveller-Cloud' + '.log'
print 'Logging to ' + logfilename
logging.basicConfig(filename=logfilename,filemode='w',level=logging.INFO,format='%(asctime)s %(message)s')
initialloggystring = 'New scan started.' + loggydatestamp
print initialloggystring
logging.info(initialloggystring)
errorcount = 0

print '\nLogging in...'
#Security Block -- Logging in with our certificates
libcloud.security.VERIFY_SSL_CERT = False
Ninefold = get_driver(Provider.NINEFOLD)
driver = Ninefold(access_token, shared_secret)
# This plays out as driver = Ninefold('YOUR Atmos Access Token HERE', 'YOUR Atmos Shared Secret HERE')

#Functions for printing the list of files and folders in cloud storage
def showcloudassets():
    try:
        containers = driver.list_containers()
        print '\nList of Containers\n'
        pprint(containers)
        print '\n'
    except:
        print "*** Error occurred: ", sys.exc_info()[0] , " ***"
        print 'Exiting...'
        sys.exit(1)	
Пример #25
0
from __future__ import print_function
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

auth_username = '******'
auth_password = '******'
auth_url = 'http://controller:5000'
project_name = 'your_project_name_or_id'
region_name = 'your_region_name'

provider = get_driver(Provider.OPENSTACK_SWIFT)
swift = provider(auth_username,
                 auth_password,
                 ex_force_auth_url=auth_url,
                 ex_force_auth_version='2.0_password',
                 ex_tenant_name=project_name,
                 ex_force_service_region=region_name)

container_name = 'fractals'
container = swift.create_container(container_name=container_name)
print(container)

print(swift.list_containers())

file_path = 'goat.jpg'
object_name = 'an amazing goat'
container = swift.get_container(container_name=container_name)
object = container.upload_object(file_path=file_path, object_name=object_name)

objects = container.list_objects()
print(objects)
Пример #26
0
 def __init__(self):
     super(LibCloudBackend, self).__init__()
     # try:
     self.driver_cls = get_driver(LIBCLOUD_DRIVER)
     self.driver = self.driver_cls(*CREDENTIALS)
     self.container = self.driver.get_container(CONTAINER_NAME)
Пример #27
0
def run():
    usage = 'usage: %prog --username=<api username> --key=<api key> [options]'
    parser = OptionParser(usage=usage)
    parser.add_option('--provider', dest='provider', default='CLOUDFILES_US',
                      help='Provider to use')
    parser.add_option('--region', dest='region', default=None,
                      help='Region to use if a Libcloud driver supports \
                        multiple regions (e.g. ORD for CloudFiles provider)')
    parser.add_option('--username', dest='api_username',
                      help='API username')
    parser.add_option('--key', dest='api_key',
                      help='API key')
    parser.add_option('--restore', dest='restore', action="store_true",
                      help='Restore from')
    parser.add_option('--container-name', dest='container_name',
                      default='file_syncer',
                      help='Name of the container storing the files')
    parser.add_option('--directory', dest='directory',
                      help='Local directory to sync')
    parser.add_option('--cache-path', dest='cache_path',
                      default=os.path.expanduser('~/.file_syncer'),
                      help='Directory where a settings and cached manifest ' +
                           'files are stored')
    parser.add_option('--concurrency', dest='concurrency', default=10,
                      help='File upload concurrency')
    parser.add_option('--exclude', dest='exclude',
                      help='Comma separated list of file name patterns to ' +
                           'exclude')
    parser.add_option('--log-level', dest='log_level', default='INFO',
                      help='Log level')
    parser.add_option('--delete', dest='delete', action='store_true',
                      help='delete extraneous files from dest containers',
                      default=False)
    parser.add_option('--auto-content-type', dest='auto_content_type',
                      default=False, action='store_true',
                      help='Don\'t automatically specify \'application/' +
                           'octet-stream\' content-type for every file. Use ' +
                           'automatic file type detection based on the file ' +
                           'extension')
    parser.add_option('--ignore-symlinks', dest='ignore_symlinks',
                      default=False, action='store_true',
                      help='Don\'t visit directories pointed to by ' +
                      'symlinks, on systems that support them')

    (options, args) = parser.parse_args()

    for option_name, key in REQUIRED_OPTIONS:
        if not getattr(options, key, None):
            raise ValueError('Missing required argument: ' + option_name)

    # Set up provider
    if options.provider not in SUPPORTED_PROVIDERS:
        raise ValueError('Invalid provider: %s. Valid providers are: %s' %
                         (options.provider, ', '.join(SUPPORTED_PROVIDERS)))

    provider = PROVIDER_MAP[options.provider]

    # Set up logger
    log_level = options.log_level.upper()

    if log_level not in VALID_LOG_LEVELS:
        valid_levels = [value.lower() for value in VALID_LOG_LEVELS]
        raise ValueError('Invalid log level: %s. Valid log levels are: %s' %
                         (options.log_level, ', ' .join(valid_levels)))

    level = getattr(logging, log_level, 'INFO')
    logger = get_logger(handler=logging.StreamHandler(), level=level)

    directory = os.path.expanduser(options.directory)
    exclude_patterns = options.exclude or ''
    exclude_patterns = exclude_patterns.split(',')

    syncer = FileSyncer(directory=directory,
                        provider_cls=get_driver(provider),
                        provider=provider,
                        region=options.region,
                        username=options.api_username,
                        api_key=options.api_key,
                        container_name=options.container_name,
                        cache_path=options.cache_path,
                        exclude_patterns=exclude_patterns,
                        logger=logger,
                        concurrency=int(options.concurrency),
                        auto_content_type=options.auto_content_type,
                        ignore_symlinks=options.ignore_symlinks)
    if options.restore:
        syncer.restore()
    else:
        syncer.sync(options.delete)
Пример #28
0
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver

access_key = 'XXXXXX'
secret_key = 'YYYYYY'

cls = get_driver(Provider.AURORAOBJECTS)
driver = cls(access_key, secret_key)
Пример #29
0
 def _get_driver(cls, name):
     """Wrapper for libcloud's get_driver for testing."""
     return get_driver(name)
Пример #30
0
 def make_driver(self):
     provider = getattr(Provider, self.provider_name)
     return get_driver(provider)(self.username, self.secret, region=self.region)