def __init__(self, conf): # DONE, CLOUDFILES COMPLIANT if conf is None: try: auth_user = config.get("cf", "apiuser") auth_key = config.get("cf", "apikey") self.container = config.get("cf", "container") try: region_name = config.get("cf", "region_name") except ConfigParser.NoOptionError: region_name = DEFAULT_RACKSPACE_LOCATION except ConfigParser.NoOptionError: log.error("Configuration file not available.") log.info("Use 'bakthat configure' to create one.") return else: auth_user = conf.get("apiuser") auth_key = conf.get("apikey") self.container = conf.get("container") region_name = conf.get("region_name", DEFAULT_RACKSPACE_LOCATION) if region_name == "dfw" or region_name == "ord": self.con = cloudfiles.get_connection(auth_user, auth_key, authurl = "https://identity.api.rackspacecloud.com/v1.0/") else: self.con = cloudfiles.get_connection(auth_user, auth_key, authurl = "https://lon.identity.api.rackspacecloud.com/v1.0/")
def open_connection_with_credentials(username,api_key,auth_url,verbose=0): if auth_url == "uk" or auth_url == "UK": auth_url="https://lon.auth.api.rackspacecloud.com/v1.0" connection = cloudfiles.get_connection(username,api_key,authurl=auth_url) else: auth_url="https://auth.api.rackspacecloud.com/v1.0" connection = cloudfiles.get_connection(username,api_key,authurl=auth_url) return connection
def __init__(self, queue, config, opt, state, l, i): """Call parent __init__ and set queue to read work from""" threading.Thread.__init__(self) self.queue = queue self.l = l self.i = i self.opt = opt self.config = config Statsd.config = config self.state = state self.source = cloudfiles.get_connection(config.get(opt.source, "user"), config.get(opt.source, "password"), authurl=config.get(opt.source, "authurl"), timeout=30) self.destination = cloudfiles.get_connection(config.get(opt.destination, "user"), config.get(opt.destination, "password"), authurl=config.get(opt.destination, "authurl"), timeout=30) self.quit = threading.Event()
def __get_container__(): username = os.environ['RACKSPACE_USERNAME'] api_key = os.environ['RACKSPACE_KEY'] conn = cloudfiles.get_connection(username, api_key) container = conn.create_container(os.environ['RACKSPACE_CONTAINER']) container.make_public() return container
def __init__(self, username, api_key, region=None, servicenet=False, timeout=4): """Initialize connection to Rackspace Cloud Files. @param username: Rackspace Cloud username @param api_key: Rackspace Cloud api_key @param region: Try passing "us" for US Auth Service, and "uk" UK Auth Service; omit parameter to use library default. @servicenet: If True, Rackspace ServiceNet network will be used to access Cloud Files. @timeout: Connection timeout in seconds. (Default: 4) """ self._connParams = {} self._connParams['username'] = username self._connParams['api_key'] = api_key if region is not None: try: authurl = getattr(cloudfiles, '%s_authurl' % str(region)) self._connParams['authurl'] = authurl except: raise Exception("Invalid region code: %s" % str(region)) if servicenet: self._connParams['servicenet'] = True self._connParams['timeout'] = timeout self._conn = cloudfiles.get_connection(**self._connParams)
def main(argv): user = None key = None delete = False try: opts, args = getopt.getopt(argv, "hu:k:c:d", ["help", "user", "key", "container", "delete"]) except getopt.GetoptError: Sync.usage() sys.exit(2) for opt, arg in opts: logging.debug("Processing opt: " + opt) if opt in ("-h", "--help"): Sync.usage(sys.argv[0], args) sys.exit() elif opt in ("-u", "--user"): user = arg elif opt in ("-k", "--key"): key = arg elif opt in ("-d", "--delete"): delete = True print "Authenticating..." conn = cloudfiles.get_connection(user, key) cfsync = Sync(conn) # cfsync.list_files(args[1]) if delete: cfsync.clear(args[0]) cfsync.upload(args[0], args[1])
def upload_to_dho(dho_user, dho_key, backup_loc): conn = cloudfiles.get_connection( username=dho_user, api_key=dho_key, authurl='https://objects-us-west-1.dream.io/auth', ) container = create_container(conn) for category in os.listdir(backup_loc): category_path = os.path.join(backup_loc, category) for section in os.listdir(category_path): section_path = os.path.join(category_path, section) for file_name in os.listdir(section_path): file_path = os.path.join(section_path, file_name) print "uploading " + file_path obj = container.create_object(file_path) uploaded = False i = 0 while i <= 4 and uploaded == False: try: obj.load_from_filename(file_path) uploaded = True except ssl.SSLError: if i < 4: print "Failed to upload " + file_path + ", trying again" i = i + 1 else: print "Failed to upload 5 times, aborting" sys.exit(1)
def sync_files(self): self.conn = cloudfiles.get_connection(self.USERNAME, self.API_KEY, servicenet=self.USE_SERVICENET) try: self.container = self.conn.get_container(self.STATIC_CONTAINER) except cloudfiles.errors.NoSuchContainer: self.container = self.conn.create_container(self.STATIC_CONTAINER) if not self.container.is_public(): self.container.make_public() # if -w option is provided, wipe out the contents of the container if self.wipe: if self.test_run: print "Wipe would delete %d objects." % self.container.object_count else: print "Deleting %d objects..." % self.container.object_count for cloud_obj in self.container.get_objects(): self.container.delete_object(cloud_obj.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count)
def main(): user = "******" key = "Rn1qOp0IQkxcT1FCDezaqxmpGmIUbw7bkpXeF31J" authurl = "http://127.0.0.1:80/auth" authurl = "http://192.168.59.103:80/auth" conn = cloudfiles.get_connection(username=user, api_key=key, authurl=authurl) print("Create test_container") test_container = conn.create_container('test_container') print("List all containers") print(conn.get_all_containers()) print("Put test_object in test_container") test_object = test_container.create_object("test_object") test_object.content_type = "text/plain" test_object.load_from_filename("/etc/ceph/ceph.conf") print("List all objects in test_container") print(test_container.get_objects()) print("Download test_object") test_object = test_container.get_object("test_object") test_object.save_to_filename('./test_object') print("Delete test_object") test_container.delete_object("test_object") print("Delete test_container") conn.delete_container("test_container")
def collect(user, key, rule, container='', dryrun=False): """ Connects to rackspace with the user and the key and crawls every container applying the rule to each cloudfile object. If the rule applies, i.e. returns True, the object is deleted. If a container name is passed to the parameter 'container', only that container will be crawled. If dryrun is True, the objects will not be deleted. The function returns a list with the names of the objects to which the rule was successfully applied. """ conn = cloudfiles.get_connection(user, key) if container: containers = [conn.get_container(container)] else: containers = conn.get_all_containers() deleted = [] for cont in containers: for obj in progress.bar(cont.get_objects(), label="Removing Objects"): if rule.apply(obj): if not dryrun: cont.delete_object(obj.name) deleted.append(obj.name) return deleted
def get_rackspace_connection(): """Returns a new Rackspace connection. Returns: new Rackspace connection """ return cloudfiles.get_connection(RS_USERNAME, RS_APIKEY)
def _get_connection(self): if not hasattr(self.local_cache, 'connection'): connection = cloudfiles.get_connection(self.username, self.api_key, **self.connection_kwargs) self.local_cache.connection = connection return self.local_cache.connection
def swift_api(request): LOG.debug('object store connection created using token "%s"' ' and url "%s"' % (request.session['token'], url_for(request, 'object-store'))) auth = SwiftAuthentication(url_for(request, 'object-store'), request.session['token']) return cloudfiles.get_connection(auth=auth)
def __init__(self, queue, config, opt, state, l): """Call parent __init__ and set queue to read work from""" threading.Thread.__init__(self) self.queue = queue self.l = l self.opt = opt self.config = config self.state = state if opt.source: # source may not be set if we're deleting self.source = cloudfiles.get_connection(config.get(opt.source, "user"), config.get(opt.source, "password"), authurl=config.get(opt.source, "authurl"), timeout=30) else: self.source = None self.destination = cloudfiles.get_connection(config.get(opt.destination, "user"), config.get(opt.destination, "password"), authurl=config.get(opt.destination, "authurl"), timeout=30) self.quit = threading.Event()
def swift_api(request): LOG.debug('object store connection created using token "%s"' ' and url "%s"' % (request.session['token'], url_for(request, 'swift'))) auth = SwiftAuthentication(url_for(request, 'swift'), request.session['token']) return cloudfiles.get_connection(auth=auth)
def handle(self, *args, **options): conn = cloudfiles.get_connection( username=settings.CUMULUS['USERNAME'], api_key=settings.CUMULUS['API_KEY'], authurl=settings.CUMULUS['AUTH_URL']) if args: containers = [] for container_name in args: try: container = conn.get_container(container_name) except cloudfiles.errors.NoSuchContainer: raise CommandError("Container does not exist: %s" % container_name) containers.append(container) else: containers = conn.get_all_containers() opts = ['name', 'count', 'size', 'uri'] for container in containers: info = { 'name': container.name, 'count': container.object_count, 'size': container.size_used, 'uri': container.public_uri() if container.is_public() else "NOT PUBLIC", } output = [str(info[o]) for o in opts if options.get(o)] if not output: output = [str(info[o]) for o in opts] print ', '.join(output) if not containers: print 'No containers found.'
def __init__(self, *args, **kwargs): "Set up the CloudFiles connection and grab the container." super(Client, self).__init__(*args, **kwargs) container_name = msettings['CLOUDFILES_CONTAINER'] username = msettings['CLOUDFILES_USERNAME'] key = msettings['CLOUDFILES_API_KEY'] if not container_name: raise ImproperlyConfigured( "CLOUDFILES_CONTAINER is a required setting.") if not username: raise ImproperlyConfigured( "CLOUDFILES_USERNAME is a required setting.") if not key: raise ImproperlyConfigured( "CLOUDFILES_API_KEY is a required setting.") self.conn = cloudfiles.get_connection(username, key) self.container = self.conn.create_container(container_name) if not self.container.is_public(): self.container.make_public()
def application(environ, start_response): from datetime import datetime our = pytz.timezone('Asia/Novosibirsk') i = datetime.now(our) _data = urlparse.parse_qs(environ["QUERY_STRING"]) username = urlparse.parse_qs(environ["QUERY_STRING"])['ceph_api_user'][0] api_key = urlparse.parse_qs(environ["QUERY_STRING"])['ceph_api_key'][0] api_key = urllib.unquote(api_key).decode('utf8') container_name = "records" record = urlparse.parse_qs(environ["QUERY_STRING"])['recording'][0] record = record[1:] record = i.strftime('%Y%m%d-%H%M%S') + "_" + record conn = cloudfiles.get_connection( username=username, api_key=api_key, authurl=authurl, ) length = int(environ["CONTENT_LENGTH"]) body = "Hello world" buf = environ['wsgi.input'].read(length) testuser = conn.create_container(container_name) obj = testuser.create_object(record) obj.write(buf) start_response("200 Ok", [("Content-Type", "text/html"), ("Content-Length", str(len(body)))]) return [body]
def __init__(self, **kwargs): self.param_container = kwargs.get('cloudfiles_container') self.param_user = kwargs.get('cloudfiles_user') self.param_api_key = kwargs.get('cloudfiles_api_key') self.param_host = kwargs.get('cloudfiles_host') self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet') # the Mime Type webm doesn't exists, let's add it mimetypes.add_type("video/webm", "webm") if not self.param_host: _log.info('No CloudFiles host URL specified, ' 'defaulting to Rackspace US') self.connection = cloudfiles.get_connection( username=self.param_user, api_key=self.param_api_key, servicenet=True if self.param_use_servicenet == 'true' or \ self.param_use_servicenet == True else False) _log.debug('Connected to {0} (auth: {1})'.format( self.connection.connection.host, self.connection.auth.host)) if not self.param_container == \ self.connection.get_container(self.param_container): self.container = self.connection.create_container( self.param_container) self.container.make_public(ttl=60 * 60 * 2) else: self.container = self.connection.get_container( self.param_container) _log.debug('Container: {0}'.format(self.container.name)) self.container_uri = self.container.public_ssl_uri()
def backup_database(): # First, generate the filename we will be using from the current time in UTC timestamp = datetime.utcnow().isoformat() timestamp = timestamp.replace(":", "-") chdir("/tmp") filename = "newdjangosite-prod-db-{0}.dump".format(timestamp) file = open(filename, "wb") # Now create the backup call(["pg_dump", "--format=custom", "newdjangosite_prod"], stdout=file) file.close() # Now connection to Cloud Files connection = get_connection(username="******", api_key=API_KEY, servicenet=True) backup_container = connection.get_container("database-backups") # Now upload the new backup current_backup = backup_container.create_object(filename) current_backup.load_from_filename(filename) # Now delete the backup file we created locally remove(filename) # Now prune the backups if needed backup_objects = backup_container.get_objects() while len(backup_objects) > 168: # Keep one backup an hour for a week (168 = 24 * 7) # We have too many backups, delete the oldest one. # The oldest one has a file name that is less than all other objects. oldest = backup_objects[0].name for object in backup_objects: if object.name < oldest: oldest = object backup_container.delete_object(oldest) backup_objects = backup_container.get_objects()
def do_login_uk(self, user, token): try: self.conn = cloudfiles.get_connection( user, token, authurl = cloudfiles.uk_authurl) except Exception as e: print("Login failed")
def __init__(self, **kwargs): self.param_container = kwargs.get("cloudfiles_container") self.param_user = kwargs.get("cloudfiles_user") self.param_api_key = kwargs.get("cloudfiles_api_key") self.param_host = kwargs.get("cloudfiles_host") self.param_use_servicenet = kwargs.get("cloudfiles_use_servicenet") # the Mime Type webm doesn't exists, let's add it mimetypes.add_type("video/webm", "webm") if not self.param_host: _log.info("No CloudFiles host URL specified, " "defaulting to Rackspace US") self.connection = cloudfiles.get_connection( username=self.param_user, api_key=self.param_api_key, servicenet=True if self.param_use_servicenet == "true" or self.param_use_servicenet == True else False, ) _log.debug("Connected to {0} (auth: {1})".format(self.connection.connection.host, self.connection.auth.host)) if not self.param_container == self.connection.get_container(self.param_container): self.container = self.connection.create_container(self.param_container) self.container.make_public(ttl=60 * 60 * 2) else: self.container = self.connection.get_container(self.param_container) _log.debug("Container: {0}".format(self.container.name)) self.container_uri = self.container.public_ssl_uri()
def check_auth_password(self, username, password): try: self._set_conn(cloudfiles.get_connection(username, password)) # authurl='https://auth.stg.swift.racklabs.com/auth')) return paramiko.AUTH_SUCCESSFUL except: return paramiko.AUTH_FAILED
def erase(args): "erase a remote file from a container" connection = cloudfiles.get_connection( RACKSPACE_USERNAME, RACKSPACE_API_KEY, ) m = '\033[1;31mAre you \033[1;37m100%\033[1;31m ' \ 'sure you wanna delete \033[1;32m"{0} ?"\033[0m (y/n)' container = connection.create_container(args.from_container) try: obj = container.get_object(args.object) question = m.format('%s (%s)' % ( obj.name, convert_bytes(obj.size))) agreed = decided = bool(args.imsure) while not decided and not agreed: i = raw_input(question).strip().lower() decided = i in ['y', 'n'] agreed = i == 'y' if agreed: obj.purge_from_cdn(email=OUR_EMAIL) container.delete_object(obj.name) else: print print "Ok then, never mind" except NoSuchObject: print 'There is no such object "{0}" in the container "{1}"'.format( args.object, args.from_container, )
def main(argv): user = None key = None delete = False try: opts, args = getopt.getopt( argv, "hu:k:c:d", ["help", "user", "key", "container", "delete"]) except getopt.GetoptError: Sync.usage() sys.exit(2) for opt, arg in opts: logging.debug("Processing opt: " + opt) if opt in ("-h", "--help"): Sync.usage(sys.argv[0], args) sys.exit() elif opt in ("-u", "--user"): user = arg elif opt in ("-k", "--key"): key = arg elif opt in ("-d", "--delete"): delete = True print "Authenticating..." conn = cloudfiles.get_connection(user, key) cfsync = Sync(conn) # cfsync.list_files(args[1]) if delete: cfsync.clear(args[0]) cfsync.upload(args[0], args[1])
def fetch_newest_object(args): "retrieves the most recent backup file" fullpath = abspath(expanduser(args.to or '.')) connection = cloudfiles.get_connection( RACKSPACE_USERNAME, RACKSPACE_API_KEY, ) container = connection.create_container(args.from_container) remote_backup_items = container.list_objects_info() remote_backup_items.sort(key=lambda x: x['last_modified']) print "found %d items in the container '%s'" % ( len(remote_backup_items), args.from_container, ) # now we have the newest being the latest newest = remote_backup_items[-1] backup = container.get_object(newest['name']) print "the latest object is %s" % newest['name'] save_to = args.to and fullpath or newest['name'] print "saving to %s" % save_to progress_callback = progress_for('downloading') progress_callback(0, 0, False) backup.save_to_filename(save_to, progress_callback)
def upload_files(username, key, service_net, auth_url, container_name, files, recursive, prefix, quiet): try: # Attempt a connection conn = cloudfiles.get_connection(username = username, api_key = key, servicenet = service_net, authurl = auth_url) except cloudfiles.errors.AuthenticationFailed: try: # Switch to the other authurl and try again if auth_url == cloudfiles.us_authurl: auth_url = cloudfiles.uk_authurl else: auth_url = cloudfiles.us_authurl conn = cloudfiles.get_connection(username = username, api_key = key, servicenet = service_net, authurl = auth_url) except cloudfiles.errors.AuthenticationFailed: # Still didn't work sys.stderr.write('Authentication failed!\n') return except: sys.stderr.write('API connection failed!\n') return try: cont = conn.get_container(container_name) except cloudfiles.errors.NoSuchContainer: cont = conn.create_container(container_name) except: sys.stderr.write('Failed to create a container called "%s"\n' % (container_name)) else: while len(files) > 0: f = files.pop(0) if os.path.isfile(f): destname = f if not recursive: destname = os.path.basename(f) try: if not quiet: sys.stdout.write('Uploading "%s"...\n' % (destname)) obj = cont.create_object('%s%s' % (prefix, destname)) obj.load_from_filename(f) except: sys.stderr.write(' Upload of "%s" failed!\n' % (f)) elif f not in ('.', '..'): if recursive: for filename in glob.glob("%s/*" % (f)): files.append(filename) else: sys.stderr.write('Ignoring "%s" because it\'s a directory\n' % (f))
def handle(self, *args, **options): """Main""" if len(args) != 1: raise CommandError('Usage: %s' % USAGE) container_name = args[0] is_yes = options.get('is_yes') num_workers = int(options.get('workers')) self.batch_size = int(options.get('batch_size')) print('Connecting') conn = cloudfiles.get_connection(username=settings.CUMULUS['USERNAME'], api_key=settings.CUMULUS['API_KEY']) container = conn.get_container(container_name) if not is_yes: is_ok = raw_input('Permanently delete container %s? [y|N]' % container) if not is_ok == 'y': raise CommandError('Aborted') queue = multiprocessing.Queue() print('Listing objects in container') while not self.is_fetch_done and queue.qsize() < (num_workers * 2): sys.stdout.write('\rQueued batches: %d. Need: %d ' % (queue.qsize(), (num_workers * 2))) sys.stdout.flush() self.fetch_more(container, queue) print('Deleting objects from container') procs = [ multiprocessing.Process(target=delete, args=(container, queue)) for _ in range(num_workers) ] for proc in procs: proc.start() print('') while not queue.empty(): sys.stdout.write('\rItems remaining: %d ' % (queue.qsize() * self.batch_size)) sys.stdout.flush() time.sleep(1) if not self.is_fetch_done: self.fetch_more(container, queue) print('Container empty. Waiting for final threads to finish.') for proc in procs: proc.join() print('Deleting container') conn.delete_container(container)
def __init__(self, username, api_key, container): """ Here we set up the connection and select the user-supplied container. If the container isn't public (available on Limelight CDN), we make it a publicly available container. """ self.connection = cloudfiles.get_connection(username, api_key) self.container = self.connection.get_container(container) if not self.container.is_public(): self.container.make_public()
def connectCloud(setting, nodeid): '''Establish connection.''' try: user = setting.nodeInfo[nodeid].accesskey key = setting.nodeInfo[nodeid].secretkey return cloudfiles.get_connection(username=user, api_key=key, timeout=666) except: return False
def _get_connection(self): if not hasattr(self, '_connection'): self._connection = cloudfiles.get_connection( username=self.username, api_key=self.api_key, authurl=self.auth_url, timeout=self.timeout, servicenet=self.use_servicenet, **self.connection_kwargs) return self._connection
def __init__(self): """ Here we set up the connection and select the user-supplied container. If the container isn't public (available on Limelight CDN), we make it a publicly available container. """ self.connection = cloudfiles.get_connection(CLOUDFILES_USERNAME, CLOUDFILES_API_KEY) self.container = self.connection.get_container(CLOUDFILES_CONTAINER) if not self.container.is_public(): self.container.make_public()
def open_connection_with_configfile(config_file='~/.pycflogin',verbose=0): if verbose >=2: print '-Debug- open_connection_with_configfile - START' if verbose >=1: print '-Debug- config_file:', config_file config_file = os.path.expanduser(config_file) if verbose >=2: print '-Debug exapanded config_file:', config_file # Read the config file config = ConfigParser.ConfigParser() if os.path.exists(config_file): config.read(config_file) # get the username and API username = config.get('account', 'username') apikey = config.get('account', 'apikey') location = config.get('account', 'location') if location == "uk" or location == "UK": auth_url="https://lon.auth.api.rackspacecloud.com/v1.0" else: auth_url="https://auth.api.rackspacecloud.com/v1.0" if verbose >= 1: print '-Info- Logging in as', username if verbose >= 2: print '-Debug- Using AUTH-URL as', auth_url else: if verbose >= 1: print '-Info- %s file does not exists' % config_file print '-Info- No existing configuration file found %s , would you like to create one?\n' % config_file answer = raw_input('Please enter yes to provide new credentials or no to exit. [yes/no]:') if answer == 'yes' or answer == 'YES': username = raw_input('username:'******'api_key:') location = raw_input('location [us/uk]:') if (location == 'uk' or location == 'UK'): auth_url="https://lon.auth.api.rackspacecloud.com/v1.0" else: auth_url="https://auth.api.rackspacecloud.com/v1.0" if verbose >= 1: print '-Info- Logging in as', username if verbose >= 2: print '-Debug- Using AUTH-URL as', auth_url if verbose >= 1: print '-Info Saving credentials in %s file for future use' % config_file fo = open(config_file, "wb") fo.write("[account]\n") fo.write("username="******"\n") fo.write("apikey=") fo.write(apikey) fo.write("\n") fo.write("location=") fo.write(location) fo.write("\n") fo.close() if verbose >= 1: print '-Debug- Credentials saved to the %s file' % config_file else: exit() connection = cloudfiles.get_connection(username,apikey,authurl=auth_url) return connection
def getStats(self): """ Query Rackspace and return stats. @return: Dictionary of stats. """ conn = cloudfiles.get_connection(self.username, self.api_key) bucket = conn.get_container(self.container) return { 'rackspace_containercount': bucket.object_count, 'rackspace_containersize': bucket.size_used }
def _get_connection(self): """Return native connection object.""" kwargs = {"username": self.account, "api_key": self.secret_key} # Only add kwarg for servicenet if True because user could set # environment variable 'RACKSPACE_SERVICENET' separately. if self.servicenet: kwargs["servicenet"] = True if self.authurl: kwargs["authurl"] = self.authurl return cloudfiles.get_connection(**kwargs)
def sendImage(self, filename): fname = os.path.split(filename)[1] conn = cloudfiles.get_connection(self.username, self.apikey, servicenet=False, authurl=self.authurl, timeout=15) container = conn.get_container('public') my_dog = container.create_object('***/*****/' + fname) my_dog.load_from_filename(filename) print "OK" #print my_dog return "http://******************************" + my_dog.name
def sync_files(self): self.conn = cloudfiles.get_connection(username = self.USERNAME, api_key = self.API_KEY, authurl = self.AUTH_URL, servicenet=self.USE_SERVICENET) try: self.container = self.conn.get_container(self.STATIC_CONTAINER) except cloudfiles.errors.NoSuchContainer: self.container = self.conn.create_container(self.STATIC_CONTAINER) if not self.container.is_public(): self.container.make_public() # if -w option is provided, wipe out the contents of the container if self.wipe: if self.test_run: print "Wipe would delete %d objects." % self.container.object_count else: print "Deleting %d objects..." % self.container.object_count for cloud_obj in self.container.get_objects(): self.container.delete_object(cloud_obj.name) if self.verbosity > 1: print "Deleted %s" % cloud_obj.name if self.verbosity > 1: print "Retreiving cloud file metadata" self.cloudfile_info_list = self.container.list_objects_info() # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally if (not self.add_only) and (not self.no_delete): self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) if self.verbosity > 1: print "CDN public uri: %s" % self.container.public_uri() print "CDN public ssl uri: %s" % self.container.public_ssl_uri() print "CDN public streaming uri: %s" % self.container.public_streaming_uri() print "CDN TTL: %s" % self.container.cdn_ttl print "CDN Size: %s" % self.container.size_used
def connectToClouds(): """ Open connections to S3 and Cloud Files """ s3Conn = None cfConn = None try: ## boto reads from /etc/boto.cfg (or ~/boto.cfg) s3Conn = boto.connect_s3() ## the cloud files library doesn't automatically read from a file, so we handle that here: cfConfig = ConfigParser.ConfigParser() cfConfig.read('/etc/cloudfiles.cfg') cfConn = cloudfiles.get_connection(cfConfig.get('Credentials','username'), cfConfig.get('Credentials','api_key')) except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError), err: raise MultiCloudMirrorException("Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err))
def _get_connection(self): """Return native connection object.""" kwargs = { 'username': self.account, 'api_key': self.secret_key, } # Only add kwarg for servicenet if True because user could set # environment variable 'RACKSPACE_SERVICENET' separately. if self.servicenet: kwargs['servicenet'] = True if self.authurl: kwargs['authurl'] = self.authurl return cloudfiles.get_connection(**kwargs) # pylint: disable=W0142
def handle(self, *args, **options): """Main""" if len(sys.argv) != 3: raise CommandError('Usage: %s' % USAGE) container_name = sys.argv[2] print('Creating container: %s' % container_name) conn = cloudfiles.get_connection(username=settings.CUMULUS['USERNAME'], api_key=settings.CUMULUS['API_KEY']) container = conn.create_container(container_name) container.make_public() print('Done')
def upload_to_dho(dho_user, dho_key, backup_loc): conn = cloudfiles.get_connection( username=dho_user, api_key=dho_key, authurl='https://objects.dreamhost.com/auth', ) container = create_container(conn) for category in os.listdir(backup_loc): category_path = os.path.join(backup_loc, category) for section in os.listdir(category_path): section_path = os.path.join(category_path, section) for file_name in os.listdir(section_path): file_path = os.path.join(section_path, file_name) print "uploading " + file_path obj = container.create_object(file_path) obj.load_from_filename(file_path)
def __init__(self, container_name, verbose, dry_run=False, forward=0, max_attempts=5): self.verbose = verbose self.dry_run = dry_run self.rp_conn = cloudfiles.get_connection(settings.RACKSPACE_USER, settings.RACKSPACE_API_KEY) self.start_time = datetime.datetime.now() self.container = self.rp_conn.get_container(container_name) self.max_attempts = max_attempts if forward == 0: self.start_time = datetime.datetime.now() else: self.start_time = datetime.datetime.now() + datetime.timedelta( days=forward) super(RackspaceUtil, self).__init__()
def sync_files(self): self.conn = cloudfiles.get_connection(username=self.USERNAME, api_key=self.API_KEY, authurl=self.AUTH_URL, servicenet=self.USE_SERVICENET) try: self.container = self.conn.get_container(self.STATIC_CONTAINER) except cloudfiles.errors.NoSuchContainer: self.container = self.conn.create_container(self.STATIC_CONTAINER) if not self.container.is_public(): self.container.make_public() # if -w option is provided, wipe out the contents of the container if self.wipe: if self.test_run: print "Wipe would delete %d objects." % self.container.object_count else: print "Deleting %d objects..." % self.container.object_count for cloud_obj in self.container.get_objects(): self.container.delete_object(cloud_obj.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count if self.CONTAINER_PURGE: self.container.purge_from_cdn(','.join( self.PURGE_NOTIFICATION_LIST)) if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count)
def setUp(self): if not hasattr(self, 'username'): cls = self.__class__ if not all([ 'RCLOUD_API_KEY' in os.environ, 'RCLOUD_API_USER' in os.environ ]): print "env RCLOUD_API_USER or RCLOUD_API_KEY not found." sys.exit(1) cls.username = os.environ['RCLOUD_API_USER'] cls.api_key = os.environ['RCLOUD_API_KEY'] cls.auth_url = os.environ.get('RCLOUD_AUTH_URL') cls.cnx = CloudFilesFS(self.username, self.api_key, authurl=self.auth_url) cls.conn = cloudfiles.get_connection(self.username, self.api_key, authurl=self.auth_url) self.cnx.mkdir("/ftpcloudfs_testing") self.cnx.chdir("/ftpcloudfs_testing") self.container = self.conn.get_container('ftpcloudfs_testing')
def connectToClouds(): """ Open connections to S3 and Cloud Files """ s3Conn = None cfConn = None try: ## boto reads from ~/.aws/credentials profiles session = boto3.Session(profile_name='jwp-webteam') s3Conn = session.resource('s3') ## the cloud files library doesn't automatically read from a file, so we handle that here: cfConfig = ConfigParser.ConfigParser() cfConfig.read('/Users/sarahgray/cloudfiles.cfg') cfConn = cloudfiles.get_connection( cfConfig.get('Credentials', 'username'), cfConfig.get('Credentials', 'api_key')) except (NoSectionError, NoOptionError, MissingSectionHeaderError, ParsingError), err: raise MultiCloudMirrorException( "Error in reading Cloud Files configuration file (/etc/cloudfiles.cfg): %s" % (err))