def main(username, project, list):
pyrax.set_setting('identity_type', 'rackspace')
with open(os.path.expanduser('~/.bugminion'), 'r') as f:
conf = json.loads(f.read())
pyrax.set_credentials(conf['access_key'],
conf['secret_key'],
region=conf['region'].upper())
conn = pyrax.connect_to_cloudfiles(region=conf['region'].upper())
container = conn.create_container(conf['container'])
now = datetime.datetime.now()
datestamp = '%04d%02d%02d' %(now.year, now.month, now.day)
with open(list) as f:
with open('%s.csv' % list, 'w') as csvfile:
csvwriter = csv.writer(csvfile, dialect='excel')
for bug in f.readlines():
bug = bug.rstrip()
try:
data = json.loads(container.get_object(
'%s-bug/%s' %(project, bug)).get())
except pyrax.exceptions.NoSuchObject:
data = {}
csvwriter.writerow([
'https://bugs.launchpad.net/nova/+bug/%s' % bug,
data.get('title', 'unknown'),
data.get('status', 'unknown'),
username])
print 'Done!'
def __init__(self, task_kwargs=None, json_config_file=None):
json_config_file = json_config_file or './deploy_settings.json'
self.cloudservers = None
self.settings = AttributeDict({})
self.fabric_env_servers = []
self.created_servers = []
task_kwargs = task_kwargs or {}
settings = self.read_settings_file(json_config_file)
for key in self.SETTINGS_KEYS:
try:
self.settings[key] = task_kwargs[key]
except KeyError:
try:
self.settings[key] = environ[key]
except KeyError:
try:
self.settings[key] = settings[key]
except KeyError:
pass
self.settings.server_count = int(self.settings.server_count)
self.settings.setdefault('ssh_user', 'root')
self.settings.setdefault('git_branch', 'master')
self.ensure_settings('rackspace_username', 'rackspace_apikey')
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_credentials(
self.settings.rackspace_username,
self.settings.rackspace_apikey)
self.cloudservers = pyrax.connect_to_cloudservers()
self.loadbalancers = pyrax.connect_to_cloud_loadbalancers()
def setup():
rax_username = get_config(p, 'rax', 'username', 'RAX_USERNAME', None)
rax_api_key = get_config(p, 'rax', 'api_key', 'RAX_API_KEY', None)
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_credentials(rax_username, rax_api_key)
region = pyrax.get_setting('region')
regions = []
if region:
regions.append(region)
else:
region_list = get_config(p, 'rax', 'regions', 'RAX_REGION', 'all',
islist=True)
for region in region_list:
region = region.strip().upper()
if region == 'ALL':
regions = pyrax.regions
break
elif region not in pyrax.regions:
sys.stderr.write('Unsupported region %s' % region)
sys.exit(1)
elif region not in regions:
regions.append(region)
return regions
def main():
# option parsing
usage = "%prog --local /var/backup --remote backup"
parser = OptionParser(usage=usage)
parser.add_option("-i", "--identity", default="rackspace",
help="Pyrax identity class")
parser.add_option("-l", "--local", help="local path to backup")
parser.add_option("-r", "--remote", help="remote container to backup to")
parser.add_option("-v", "--verbose", action="count",
help="Increase verbosity", default=0)
(options, args) = parser.parse_args()
for option in (options.local, options.remote):
if option is None:
parser.print_help()
sys.exit(1)
# Get login details from .netrc.
login, account, password = netrc().authenticators(
'pyrax.%s' % options.identity)
# Configure logging
logging.basicConfig(level=max(4-options.verbose,1)*10)
logging.info("Logging on to %s", options.identity)
pyrax.set_setting("identity_type", options.identity)
pyrax.set_credentials(login, password)
logging.info("Synchronising")
pyrax.cloudfiles.sync_folder_to_container(
options.local, options.remote, delete=True, ignore_timestamps=True)
def main():
pyrax.set_setting("identity_type", "rackspace")
creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
pyrax.set_credential_file(creds_file)
available_regions = pyrax.regions
print "Regions Available:", " ".join(list(available_regions))
region = ""
while region not in pyrax.regions:
region = raw_input("Select region: ")
cs = pyrax.connect_to_cloudservers(region)
servers = cs.servers.list()
srv_dict = {}
print "Select a server from which an image will be created."
for pos, srv in enumerate(servers):
print "%s: %s" % (pos, srv.name)
srv_dict[str(pos)] = srv.id
selection = None
while selection not in srv_dict:
if selection is not None:
print " -- Invalid choice"
selection = raw_input("Enter the number for your choice: ")
server_id = srv_dict[selection]
print
nm = raw_input("Enter a name for the image: ")
img_id = cs.servers.create_image(server_id, nm)
print "Image '%s' is being created. Its ID is: %s" % (nm, img_id)
def main(username, project, list):
pyrax.set_setting('identity_type', 'rackspace')
with open(os.path.expanduser('~/.bugminion'), 'r') as f:
conf = json.loads(f.read())
pyrax.set_credentials(conf['access_key'],
conf['secret_key'],
region=conf['region'].upper())
conn = pyrax.connect_to_cloudfiles(region=conf['region'].upper())
container = conn.create_container(conf['container'])
# Prioritize a list of bugs from an input file
now = datetime.datetime.now()
datestamp = '%04d%02d%02d' %(now.year, now.month, now.day)
with open(list) as f:
for bug in f.readlines():
bug = bug.rstrip()
triage = {'reviewer': username,
'osic': 'y'}
common.clobber_object(container,
'%s-bug/%s-%s' %(project, bug, datestamp),
json.dumps(triage, indent=4, sort_keys=True))
print 'Done!'
def __init__(self, name):
self.region, name = name.split('://')
self.basename = os.path.basename(name)
pyrax.set_setting('identity_type', 'rackspace')
with open(os.path.expanduser('~/.cloudfiles'), 'r') as f:
self.conf = json.loads(f.read())
pyrax.set_credentials(self.conf['access_key'],
self.conf['secret_key'],
region=self.region)
conn = pyrax.connect_to_cloudfiles(region=self.region.upper())
if self.region == 'dfw':
self.container_name = remote_filename(name)
else:
self.container_name = remote_filename('%s/%s' %(self.region, name))
container = conn.create_container(self.container_name)
for i in range(3):
try:
container.log_retention(True)
break
except:
pass
for info in conn.list_containers_info():
if info['name'] == self.container_name:
remote_total = info['bytes']
print ('%s Remote store %s contains %s in %d objects'
%(datetime.datetime.now(), self.region,
utility.DisplayFriendlySize(remote_total),
info['count']))
def main():
domain_name = sys.argv[1]
pyrax.set_setting('identity_type', 'rackspace')
creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
pyrax.set_credential_file(creds_file)
dns = pyrax.cloud_dns
# Create the domain name.
try:
domain = dns.find(name=domain_name)
except exc.NotFound:
domain = dns.create(name=domain_name, emailAddress='*****@*****.**', ttl=DEFAULT_TTL)
records = [{
'type': 'A',
'name': domain_name,
'data': MVPCREATOR_IP,
'ttl': DEFAULT_TTL,
}, {
'type': 'CNAME',
'name': '*.' + domain_name,
'data': domain_name,
'ttl': DEFAULT_TTL,
}]
domain.add_records(records)
def mount(volume_name, server_name, etcd):
resp = requests.get(
build_url(etcd, 'rackspace', 'credentials')
).json()
credentials = json.loads(resp['node']['value'])
username = credentials['username']
api_key = credentials['apiKey']
region = credentials['region']
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_credentials(username, api_key, region=region)
cs = pyrax.cloudservers
cbs = pyrax.cloud_blockstorage
volume = cbs.find(display_name=volume_name)
server = cs.servers.find(name=server_name)
if volume.attachments and volume.attachments[0]['server_id'] != server.id:
volume.detach()
pyrax.utils.wait_until(volume, 'status', 'available', interval=3, attempts=0)
if not volume.attachments:
volume.attach_to_instance(server, mountpoint='')
pyrax.utils.wait_until(volume, 'status', 'in-use', interval=3, attempts=0)
resp = requests.put(
build_url(etcd, 'rackspace', 'cbs', volume_name),
data={"value": volume.attachments[0]['device']}
)
def main():
'''Script execution'''
parser = argparse.ArgumentParser(description='get percent of api limit '
'of ram used')
parser.add_argument('-u', '--username', help='Rackspace Username',
required=True)
parser.add_argument('-a', '--apikey', help='Rackspace API Key',
required=True)
parser.add_argument('-m', '--maxthreshold',
help='API Percent Used Threshold, integer between '
'1-99',
required=True)
parser.add_argument('-r', '--region', help='Rackspace Regional Datacenter',
required=True)
parser.add_argument('--human',
help='Format output for humans, not Cloud Monitoring',
action='store_true')
args = parser.parse_args()
if int(args.maxthreshold) < 1 or int(args.maxthreshold) > 99:
print "You must enter a valid integer from 1-99 for maxthreshold"
sys.exit(2)
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(args.username, args.apikey)
(ram_used, ram_allowed) = getlimits(args.region)
display_usage(ram_used, ram_allowed, args.maxthreshold, args.human)
def setup():
username = os.environ.get('OS_USERNAME')
api_key = os.environ.get('OS_PASSWORD')
credentials = os.environ.get('RAX_CREDENTIALS') or os.environ.get('RAX_CREDS_FILE')
region = os.environ.get('OS_REGION_NAME')
if credentials is None:
credentails = os.path.expanduser('~/.rackspace_cloud_credentials')
try:
pyrax.set_setting('identity_type', 'rackspace')
if api_key and username:
pyrax.set_credentials(username, api_key=api_key)
elif credentials:
credentials = os.path.expanduser(credentials)
pyrax.set_credential_file(credentials)
else:
sys.stderr.write('No value in environment variable %s and/or no '
'credentials file at %s\n'
% (e.message, default_creds_file))
sys.exit(1)
except Exception, e:
sys.stderr.write("%s: %s\n" % (e, e.message))
sys.exit(1)
def __init__(self):
"""Create a logger and setup connection to Rackspace Cloudfiles."""
# Set up a logger to track what is going on in case there is a problem
self.logger = logging.getLogger("backup_client")
handle = logging.FileHandler(settings.BACKUP_LOG_FILE_LOCATION)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
handle.setFormatter(formatter)
self.logger.addHandler(handle)
self.logger.setLevel(logging.INFO)
# set name of backup file and its zipped version
self._hostname = platform.node()
self._set_backup_file_name()
self._timestamp = str(datetime.datetime.now())
# get a cloudfiles connection
try:
self.logger.info("********** Start Backup **********")
self.logger.info("Establishing connection to Cloudfiles")
# Establish a connection to Rackspace
pyrax.set_setting("identity_type", "rackspace")
self.conn = pyrax.set_credentials(
settings.CLOUDFILES_USER, settings.CLOUDFILES_API_KEY)
# Get a connection to cloudfiles for reading and writing.
self.cf = pyrax.cloudfiles
except:
msg = "Unable to establish a connection to cloudfiles"
self.logger.exception(msg)
raise CloudfileConnectionError(msg)
def load(context, path, callback):
key = (
context.config.RACKSPACE_PYRAX_REGION,
context.config.get('RACKSPACE_PYRAX_IDENTITY_TYPE','rackspace'),
context.config.RACKSPACE_PYRAX_CFG,
context.config.RACKSPACE_PYRAX_PUBLIC,
context.config.RACKSPACE_LOADER_CONTAINER
)
if key not in CONNECTIONS:
if(context.config.RACKSPACE_PYRAX_REGION):
pyrax.set_default_region(context.config.RACKSPACE_PYRAX_REGION)
pyrax.set_setting('identity_type', context.config.get('RACKSPACE_PYRAX_IDENTITY_TYPE','rackspace'))
pyrax.set_credential_file(expanduser(context.config.RACKSPACE_PYRAX_CFG))
cf = pyrax.connect_to_cloudfiles(public=context.config.RACKSPACE_PYRAX_PUBLIC)
CONNECTIONS[key] = cf.get_container(context.config.RACKSPACE_LOADER_CONTAINER)
cont = CONNECTIONS[key]
file_abspath = normalize_path(context, path)
logger.debug("[LOADER] getting from %s/%s" % (context.config.RACKSPACE_LOADER_CONTAINER, file_abspath))
try:
obj = cont.get_object(file_abspath)
if obj:
logger.debug("[LOADER] Found object at %s/%s" % (context.config.RACKSPACE_LOADER_CONTAINER, file_abspath))
else:
logger.warning("[LOADER] Unable to find object %s/%s" % (context.config.RACKSPACE_LOADER_CONTAINER, file_abspath ))
except:
callback(None)
else:
callback(obj.get())
def main():
# Log the startup
jarlog.logit('INFO', "Starting jarflyd")
# Get the config
confobj = config.GetConfig()
# Set up the identity type
pyrax.set_setting("identity_type", "rackspace")
# Set up the credentials file
cred_file = confobj.get("global", "credentials_file")
jarlog.logit('INFO', "Authenticating using cred file: %s" % cred_file)
pyrax.set_credential_file(cred_file)
# Set up the default region
globalRegion = confobj.get("global", "region")
jarlog.logit('INFO', "Setting global region to: %s" % globalRegion)
pyrax.connect_to_services(region=globalRegion)
# Start reading in the jar sections
sections = confobj.sections()
for section in sections:
if section.startswith("jar-"):
processJar(confobj, section)
def authenticate_credentials(self):
"""
This method try to authenticate with available credentials
:returns: True or False (Boolean)
"""
logger = logging.getLogger(__name__)
logger.debug('authenticating with credentials '
'(identity_type:%s, username:%s, api-key:%s, region=%s)',
self._identity_type, self._username, self._apikey, self._region)
try:
pyrax.set_setting("identity_type", self._identity_type)
pyrax.set_credentials(self._username, self._apikey,
region=self._region)
logger.info("authenticated with credentials, username: %s, "
"api-key: %s, region: %s, identity_type: %s",
self._username, self._apikey, self._region, self._identity_type)
logger.debug("user authenticated: %s", pyrax.identity.authenticated)
if pyrax.identity.authenticated:
self._token = pyrax.identity.auth_token
self._tenant_id = pyrax.identity.tenant_id
self.save_token()
return pyrax.identity.authenticated
except AuthenticationFailed:
logger.warn("cannot authenticate with credentials")
return False
def api_initialization():
pyrax.set_setting("identity_type", "rackspace")
try:
progname, username, api_key = argv
pyrax.set_credentials(username, api_key)
except ValueError:
if os.path.isfile(os.path.expanduser("~/.rackspace_cloud_credentials")):
creds_file = os.path.expanduser("~/.rackspace_cloud_credentials")
try:
pyrax.set_credential_file(creds_file)
except pexc.AuthenticationFailed:
print "The credentials located in ~/.rackspace_cloud_credentials are not valid. Please provide the correct Username and API Key below.\n"
cred_prompt()
else:
cred_prompt()
except pexc.AuthenticationFailed:
if os.path.isfile(os.path.expanduser("~/.rackspace_cloud_credentials")):
print "The provided credentials are not valid; reverting to the ~/.rackspace_cloud_credentials file."
creds_file = os.path.expanduser("~/.rackspace_cloud_credentials")
try:
pyrax.set_credential_file(creds_file)
except pexc.AuthenticationFailed:
print "The credentials located in ~/.rackspace_cloud_credentials are not valid. Please provide the correct Username and API Key below.\n"
cred_prompt()
else:
print "The provided credentials are not valid; please enter them below.\n"
cred_prompt()
def main():
pyrax.set_setting("identity_type", "rackspace")
creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
pyrax.set_credential_file(creds_file)
available_regions = pyrax.regions
print "Regions Available:", " ".join(list(available_regions))
region = ""
while region not in available_regions:
region = raw_input("Select Region: ")
cs = pyrax.connect_to_cloudservers(region)
available_images = cs.images.list()
available_flavors = cs.flavors.list()
image = make_choice(images, "Select an image: ")
flavor = make_choice(flavors, "Select a flavor: ")
base_name = raw_input('Base server name': )
server_count = int(raw_input('Number of servers to build: '))
print "Building {} servers with base name '{}', flavor '{}', and image "
"'{}'.".format(server_count, base_name, flavor.name, image.name)
def impl(context, operating_system, ram_size):
server_id = context.output.strip()
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'])
cs = pyrax.cloudservers
import code
code.interact(local=locals())
def test_set_region_setting(self):
ident = pyrax.identity
ident.region = "DFW"
pyrax.set_setting("region", "ORD")
self.assertEqual(ident.region, "DFW")
pyrax.set_setting("region", "LON")
self.assertEqual(ident.region, "LON")
def upload(self, local_dir, cf_prefix, container_name=None):
pyrax.set_setting('identity_type', 'rackspace')
try:
pyrax.set_credentials(Configuration().SWIFT_USERNAME, Configuration().SWIFT_API_KEY)
except pyrax.exceptions.AuthenticationFailed, e:
self.logger.exception(e)
raise
def update_records(self, ip_addresses):
record_types = {
'ipv4': 'A',
'ipv6': 'AAAA'
}
username = self.config_get('username')
if username is None:
raise exc.NoUsername('A username is not configured in %s' %
self.config_file)
apikey = self.config_get('apikey')
if apikey is None:
raise exc.NoApiKey('An API key is not configured in %s' %
self.config_file)
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_credentials(username, apikey)
self.dns = pyrax.cloud_dns
dns_info = self.find_dns()
for ip_type, ip in ip_addresses.iteritems():
if ip is None:
continue
if dns_info[ip_type] is None:
self.logger.info('Creating %s record for %s' %
(record_types[ip_type], dns_info['host']))
records = dns_info['domain'].add_records([
{
'type': record_types[ip_type],
'name': dns_info['host'],
'data': ip,
'ttl': 300
}
])
else:
self.logger.info('Updating %s record for %s' %
(record_types[ip_type], dns_info['host']))
dns_info[ip_type].update(data=ip)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
import pyrax
except ImportError as e:
raise BackendException("""\
Hubic backend requires the pyrax library available from Rackspace.
Exception: %s""" % str(e))
# Inform Pyrax that we're talking to Hubic
pyrax.set_setting("identity_type", "duplicity.backends.pyrax_identity.hubic.HubicIdentity")
CREDENTIALS_FILE = os.path.expanduser("~/.hubic_credentials")
if os.path.exists(CREDENTIALS_FILE):
try:
pyrax.set_credential_file(CREDENTIALS_FILE)
except Exception as e:
log.FatalError("Connection failed, please check your credentials: %s %s"
% (e.__class__.__name__, util.uexc(e)),
log.ErrorCode.connection_failed)
else:
raise BackendException("No ~/.hubic_credentials file found.")
container = parsed_url.path.lstrip('/')
self.client_exc = pyrax.exceptions.ClientException
self.nso_exc = pyrax.exceptions.NoSuchObject
self.container = pyrax.cloudfiles.create_container(container)
def master_up(key_name, credential_file="~/.rackspace_cloud_credentials"):
'''
Create a salt-master on Rackspace
Alternatively create the master using nova
'''
# Authenticate with Rackspace, use credential file
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credential_file(os.path.expanduser(credential_file))
# Shorthand
cs = pyrax.cloudservers
# Building Ubuntu 12.04 boxes with 512 MB RAM
iter_flavors = ifilter(lambda flavor: flavor.ram == 512, cs.flavors.list())
flavor_512 = iter_flavors.next()
iter_os = ifilter(lambda img: "Ubuntu 12.04" in img.name, cs.images.list())
ubu_image = iter_os.next()
master = cs.servers.create("master.ipython.org", ubu_image.id, flavor_512,
key_name=key_name)
master = pyrax.utils.wait_for_build(master, verbose=True)
env.hosts = [master.accessIPv4]
print("Master IP: {}".format(master.accessIPv4))
return master.accessIPv4
def cloud_connect():
creds_file = os.path.expanduser("~/.rackspace_cloud_credentials")
try:
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credential_file(creds_file)
except exc.AuthenticationFailed:
print "Problem with credential file ~/.rackspace_cloud_credentials"
def conn(self):
if self._conn is None:
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(os.environ["RACKSPACE_USERNAME"], os.environ["RACKSPACE_API_KEY"])
pyrax.set_setting("verify_ssl", False)
self._conn = pyrax # We love globals, don't we?!
return self._conn
def content_store_url(quiet=False):
"""
Access the public content store URL.
Respect the environment variable CONTENT_STORE_URL if it is populated.
Otherwise, find the content store load balancer and derive its public IP
via the Rackspace API.
Prints the derived URL to stdout as a side-effect unless "quiet" is set to
True.
"""
content_store_url = os.environ.get("CONTENT_STORE_URL")
domain = get("domain")
if content_store_url:
if content_store_url.endswith("/"):
content_store_url = content_store_url[:-1]
if not quiet:
print("Using content store URL: {}".format(content_store_url))
return content_store_url
elif domain:
content_store_url = "https://{}:9000".format(domain)
if not quiet:
print("Using content store URL: {}".format(content_store_url))
return content_store_url
else:
rackspace_username = get("rackspace_username")
rackspace_apikey = get("rackspace_api_key")
rackspace_region = get("rackspace_region")
instance_name = get("instance")
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_setting("region", rackspace_region)
pyrax.set_credentials(rackspace_username, rackspace_apikey)
clb = pyrax.cloud_loadbalancers
the_lb = None
content_lb_name = "deconst-{}-content".format(instance_name)
for lb in clb.list():
if lb.name == content_lb_name:
the_lb = lb
if not the_lb:
raise Exception("Content service load balancer not found")
addr = the_lb.virtual_ips[0].address
port = the_lb.port
content_store_url = "https://{}:{}".format(addr, port)
if not quiet:
print("Derived content store URL: {}".format(content_store_url))
print("If this is incorrect, set CONTENT_STORE_URL to the correct value.")
return content_store_url
def __init__(self, parsed_url):
try:
import pyrax
except ImportError:
raise BackendException("This backend requires the pyrax "
"library available from Rackspace.")
# Inform Pyrax that we're talking to Rackspace
# per Jesus Monzon (gsusmonzon)
pyrax.set_setting("identity_type", "rackspace")
conn_kwargs = {}
if not os.environ.has_key('CLOUDFILES_USERNAME'):
raise BackendException('CLOUDFILES_USERNAME environment variable'
'not set.')
if not os.environ.has_key('CLOUDFILES_APIKEY'):
raise BackendException('CLOUDFILES_APIKEY environment variable not set.')
conn_kwargs['username'] = os.environ['CLOUDFILES_USERNAME']
conn_kwargs['api_key'] = os.environ['CLOUDFILES_APIKEY']
if os.environ.has_key('CLOUDFILES_REGION'):
conn_kwargs['region'] = os.environ['CLOUDFILES_REGION']
container = parsed_url.path.lstrip('/')
try:
pyrax.set_credentials(**conn_kwargs)
except Exception, e:
log.FatalError("Connection failed, please check your credentials: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
def provision():
# Set the credentials for pyrax
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credential_file(".pyrax.cfg")
# Provision the server with Rackspace
server = pyrax.cloudservers.servers.create(
"warehouse-repository",
IMAGE_ID,
FLAVOR_ID,
)
# Store the password
fabric.state.env.password = server.adminPass
# Wait until the server is built
server = pyrax.utils.wait_for_build(server)
fabric.state.env.host_string = "root@{}".format(
filter(lambda x: "." in x, server.networks["public"])[0],
)
# Bootstrap our environment
_bootstrap_environment()
# Write out our config
with open(".rackspace.json", "w") as rs:
json.dump(
{
"host_string": fabric.state.env.host_string,
"password": fabric.state.env.password,
},
rs,
)
def get_queryset(self):
pyrax.set_setting("identity_type", "rackspace")
profile = UserProfile.get_profile(self.request.user)
username = profile.rackspace_username
api_key = profile.rackspace_api_key
pyrax.set_credentials(username, api_key)
return pyrax.cloudservers.list()
def servers_api(creds_file, region = None):
"""Sets up a pyrax instance for Cloud Servers in the given region."""
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credential_file(os.path.expanduser(creds_file), region = region)
cs = pyrax.cloudservers
return cs
def setup():
default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
env = get_config(p, 'rax', 'environment', 'RAX_ENV', None)
if env:
pyrax.set_environment(env)
keyring_username = pyrax.get_setting('keyring_username')
# Attempt to grab credentials from environment first
creds_file = get_config(p, 'rax', 'creds_file', 'RAX_CREDS_FILE', None)
if creds_file is not None:
creds_file = os.path.expanduser(creds_file)
else:
# But if that fails, use the default location of
# ~/.rackspace_cloud_credentials
if os.path.isfile(default_creds_file):
creds_file = default_creds_file
elif not keyring_username:
sys.exit('No value in environment variable %s and/or no '
'credentials file at %s' %
('RAX_CREDS_FILE', default_creds_file))
identity_type = pyrax.get_setting('identity_type')
pyrax.set_setting('identity_type', identity_type or 'rackspace')
region = pyrax.get_setting('region')
try:
if keyring_username:
pyrax.keyring_auth(keyring_username, region=region)
else:
pyrax.set_credential_file(creds_file, region=region)
except Exception as e:
sys.exit("%s: %s" % (e, e.message))
regions = []
if region:
regions.append(region)
else:
try:
# Ansible 2.3+
region_list = get_config(p,
'rax',
'regions',
'RAX_REGION',
'all',
value_type='list')
except TypeError:
# Ansible 2.2.x and below
# pylint: disable=unexpected-keyword-arg
region_list = get_config(p,
'rax',
'regions',
'RAX_REGION',
'all',
islist=True)
for region in region_list:
region = region.strip().upper()
if region == 'ALL':
regions = pyrax.regions
break
elif region not in pyrax.regions:
sys.exit('Unsupported region %s' % region)
elif region not in regions:
regions.append(region)
return regions
import config
import time
import uuid
import subprocess
import os
import sendgrid
import pyrax
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_setting("region", config.rackspace['API_REGION'])
def do_main_program() :
pyrax.set_credentials(config.rackspace['API_USERNAME'], config.rackspace['API_KEY'])
my_client_id = str(uuid.uuid4())
pq = pyrax.queues
pq.client_id = my_client_id
cf = pyrax.cloudfiles
in_container = cf.get_container(config.rackspace['API_FILES_IN'])
out_container = cf.get_container(config.rackspace['API_FILES_OUT'])
# We set the ttl and grace period to their minimum, 60 seconds.
# Get 1 at a time.
claim = pq.claim_messages(config.rackspace['API_QUEUE'], 60, 60, 1)
if claim and len(claim.messages) :
for msg in claim.messages:
def main():
"""
Challenge 6
-- Write a script that creates a CDN-enabled container in Cloud Files
"""
# Parse script parameters
p = argparse.ArgumentParser(description=("Push local objects "
"to Cloud Files"))
p.add_argument("directory",
action="store",
type=str,
metavar="[local dir]",
help="Local directory to upload content from")
p.add_argument("container",
action="store",
type=str,
metavar="[container name]",
help="Container name where content should be uploaded")
p.add_argument("-r",
"--region",
action="store",
required=False,
metavar="[region]",
type=str,
help=("Region where container should be created"
" (defaults to 'ORD'"),
choices=["ORD", "DFW", "LON", "IAD", "HKG", "SYD"],
default="ORD")
p.add_argument("-t",
"--ttl",
action="store",
required=False,
type=int,
help=("CDN TTL for the container (default '%d seconds')" %
(MIN_TTL)),
default=MIN_TTL)
p.add_argument("-f",
"--force",
action="store_true",
required=False,
help=("Permit upload to an "
"existing container"))
args = p.parse_args()
# Determine if the upload directory exists
if not os.path.isdir(args.directory):
print(
"ERROR: Specified directory (%s) does not exist, please check "
"the path and try again)" % (args.directory))
sys.exit(1)
# Determine if TTL is at least the minimum value permitted
if args.ttl < MIN_TTL:
print "ERROR: Minimum TTL permitted is %ds" % (MIN_TTL)
sys.exit(2)
# Define the authentication credentials file location and request that
# pyrax makes use of it. If not found, let the client/user know about it.
# Use a credential file in the following format:
# [rackspace_cloud]
# username = myusername
# api_key = 01234567890abcdef
# region = LON
try:
creds_file = os.path.expanduser(CONFIG_FILE)
pyrax.set_setting("identity_type", IDENTITY_TYPE)
pyrax.set_credential_file(creds_file, args.region)
except e.AuthenticationFailed:
print(
"ERROR: Authentication failed. Please check and confirm "
"that the API username, key, and region are in place "
"and correct.")
sys.exit(3)
except e.FileNotFound:
print "ERROR: Credentials file '%s' not found" % (creds_file)
sys.exit(4)
# Use a shorter Cloud Files class reference string
# This simplifies invocation later on (less typing)
cf = pyrax.cloudfiles
# Determine if container already exists, otherwise create it
try:
print "INFO: Checking if container already exists..."
cont = cf.get_container(args.container)
except:
cont = None
# Container not found, create it and CDN enable
if cont is None:
try:
print(
"INFO: Container '%s' not found, creating with TTL set "
"to %d..." % (args.container, args.ttl))
cont = cf.create_container(args.container)
cont.make_public(ttl=args.ttl)
except:
print "ERROR: Could not create container", args.container
sys.exit(5)
# Otherwise inform the user/client that the directory exists and
# determine if we can proceed (is the overwrite flag set)
else:
print("INFO: Container '%s' found with TTL set to %d" %
(cont.name, cont.cdn_ttl))
if args.force:
print "INFO: Proceeding as force flag is set"
else:
print "INFO: Force flag not set, exiting..."
sys.exit(6)
# Start the upload
print "INFO: Beginning directory/folder upload"
(upload_key, total_bytes) = cf.upload_folder(args.directory, cont)
# Inform the user of the total upload size
print("INFO: Total upload size: %d bytes (%.2f MB)" %
(total_bytes, total_bytes / 1024.0 / 1024))
# Prepare progress toolbar
sys.stdout.write("[%s]" % (" " * TOOLBAR_WIDTH))
sys.stdout.flush()
# Return to start of line, after '['
sys.stdout.write("\b" * (TOOLBAR_WIDTH + 1))
# Print the upload progress (1 second interval)
uploaded = 0
while uploaded < total_bytes:
uploaded = cf.get_uploaded(upload_key)
progress = int(ceil((uploaded * 100.0) / total_bytes / 2))
sys.stdout.write("%s" % ("=" * progress))
sys.stdout.flush()
sys.stdout.write("\b" * (progress)) # Reset progress bar
sleep(1)
print
# Upload completed, print object count and CDN URIs
objs = cf.get_container_object_names(cont)
print "INFO: Number of objects uploaded: %d" % (len(objs))
print(
"\nCDN links:\n\tHTTP: %s\n\tHTTPS: %s\n\tStreaming: %s\n\tiOS "
"streaming: %s\n" % (cont.cdn_uri, cont.cdn_ssl_uri,
cont.cdn_streaming_uri, cont.cdn_ios_uri))
print "INFO: Upload completed successfully"
def launch_node(prefix="compmodels", region="iad", node_num=1, domain="tmpnb.org"):
key_name = "team"
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(os.environ["OS_USERNAME"], os.environ["OS_PASSWORD"])
cs = pyrax.connect_to_cloudservers(region=region.upper())
# My least favorite bug in pyrax - silent errors
if(cs is None):
raise Exception("Unable to connect to given region '{}'".format(region))
# Get our base images
images = cs.list_base_images()
ubs = [image for image in images if "Ubuntu 14.04" in image.name]
user_image = [image for image in ubs if "OnMetal" in image.name][0]
proxy_image = [image for image in ubs if "PVHVM" in image.name][0]
user_server_name, proxy_server_name = name_new_nodes(prefix=prefix,
region=region.lower(),
node_num=node_num,
domain=domain)
# Launch the servers
proxy_server = cs.servers.create(proxy_server_name, image=proxy_image.id, flavor='performance2-15', key_name=key_name)
user_server = cs.servers.create(user_server_name, image=user_image.id, flavor='onmetal-compute1', key_name=key_name)
# Wait on them
print("Waiting on Proxy server")
proxy_server = pyrax.utils.wait_for_build(proxy_server, verbose=True)
print("Waiting on jupyterhub user server")
user_server = pyrax.utils.wait_for_build(user_server, verbose=True)
# Making this in case we want some JSON
node_layout = {
'jupyterhub_server': {
'private': user_server.networks['private'][0],
'public': user_server.networks['public'][0]
},
'proxy_server': {
'public': proxy_server.networks['public'][0]
}
}
inventory = '''[jupyterhub]
{user_server_name} ansible_ssh_user=root ansible_ssh_host={jupyterhub_server_public}
[proxy]
{proxy_server_name} ansible_ssh_user=root ansible_ssh_host={proxy_server_public} jupyterhub_host={jupyterhub_server_private}
'''.format(jupyterhub_server_public=user_server.accessIPv4,
jupyterhub_server_private=user_server.networks['private'][0],
proxy_server_public=proxy_server.accessIPv4,
user_server_name=user_server_name,
proxy_server_name=proxy_server_name,
)
print(inventory)
# If a separate account is used for DNS, use that instead
if("OS_DNS_USERNAME" in os.environ and "OS_DNS_PASSWORD" in os.environ):
pyrax.set_credentials(os.environ["OS_DNS_USERNAME"], os.environ["OS_DNS_PASSWORD"])
dns = pyrax.cloud_dns.find(name=domain)
dns.add_record({'type': 'A',
'name': proxy_server_name,
'ttl': 60*5,
'data': proxy_server.accessIPv4})
def main():
"""
Challenge 8
-- Write a script that will create a static webpage served out of
Cloud Files. The script must create a new container, CDN enable it,
enable it to serve an index file, create an index file object,
upload the object to the container, and create a CNAME record
pointing to the CDN URL of the container.
"""
# Parse script parameters
p = argparse.ArgumentParser(description=("Create a Cloud Files static "
"web site from a local "
"folder/directory"))
p.add_argument("directory", action="store", type=str,
metavar="[source directory]",
help="Local directory to upload content from")
p.add_argument("container", action="store", type=str,
metavar="[container name]",
help="Container name where content should be uploaded")
p.add_argument("fqdn", action="store", type=str, metavar="[cname]",
help="CNAME for CDN alias record")
p.add_argument("-r", "--region", action="store", required=False,
metavar="[region]", type=str,
help=("Region where container should be created"
" (defaults to 'ORD'"),
choices=["ORD", "DFW", "LON", "IAD", "HKG", "SYD"],
default="ORD")
p.add_argument("-t", "--cdn-ttl", action="store", required=False,
metavar="[cdn ttl]", type=int,
help=("CDN TTL for the container (default %d seconds)" %
(MIN_TTL)),
default=MIN_TTL)
p.add_argument("-i", "--index", action="store", required=False,
metavar="[static web index]", type=str,
help="Static web index file (default 'index.html')",
default="index.html")
p.add_argument("-l", "--cname-ttl", action="store", required=False,
type=int, metavar="[cname TTL]",
help=(("CNAME record TTL (in seconds) for the CDN "
"URI (default %d seconds)") % (DEFAULT_TTL)),
default=DEFAULT_TTL)
p.add_argument("-f", "--force", action="store_true",
required=False,
help="Permit upload to an existing container")
# Parse arguments (validate user input)
args = p.parse_args()
# Determine if the upload directory exists
if not os.path.isdir(args.directory):
print ("ERROR: Specified directory (%s) does not exist, please check "
"the path and try again)" % (args.directory))
sys.exit(1)
# Determine if CDN TTL is at least the minimum value permitted
if args.cdn_ttl < MIN_TTL:
print "ERROR: Minimum CDN TTL permitted is %ds" % (MIN_TTL)
sys.exit(2)
# Same again for the CNAME record
if args.cname_ttl < DEFAULT_TTL:
print "ERROR: Minimum CNAME TTL permitted is %ds" % (DEFAULT_TTL)
sys.exit(3)
# Determine if the FQDN is correctly formated (at least three segments
# separated by '.' are required).
# NOTE: This can be improved since we're not checking whether or not
# the zone in question is a valid TLD or if the string only has
# valid (alphanumeric) characters
segments = args.fqdn.split('.')
if len(segments) < 3:
print ("ERROR: FQDN string is incorrectly formatted, please check "
"and try again")
print ("Base zone/domain in the format 'example.com' will not be "
"accepted")
sys.exit(4)
# All is apparently well, define the zone/domain using the FQDN string
else:
zone_name = '.'.join(segments[-(len(segments)-1):])
# Define the authentication credentials file location and request that
# pyrax makes use of it. If not found, let the client/user know about it.
# Use a credential file in the following format:
# [rackspace_cloud]
# username = myusername
# api_key = 01234567890abcdef
# region = LON
try:
creds_file = os.path.expanduser(CONFIG_FILE)
pyrax.set_setting("identity_type", IDENTITY_TYPE)
pyrax.set_credential_file(creds_file, args.region)
except e.AuthenticationFailed:
print ("ERROR: Authentication failed. Please check and confirm "
"that the API username, key, and region are in place "
"and correct.")
sys.exit(5)
except e.FileNotFound:
print "ERROR: Credentials file '%s' not found" % (creds_file)
sys.exit(6)
# Use a shorter Cloud Files and Cloud DNS class reference strings
# This simplifies invocation later on (less typing)
cf = pyrax.cloudfiles
dns = pyrax.cloud_dns
# Grab zone list
domains = zone_list(dns)
# No zones found, inform that one needs to be created and exit
if len(domains) == 0:
print "ERROR: You have no domains/zones at this time"
print "Please create one first then try again"
sys.exit(7)
# Attempt to locate the zone extracted from FQDN string
try:
zone = [i for i in domains if zone_name in i.name][0]
except:
print "ERROR: Zone '%s' not found" % (zone_name)
print "Please check/create and try again"
sys.exit(8)
# Determine if container already exists, otherwise create it
try:
print "INFO: Checking if container already exists..."
cont = cf.get_container(args.container)
except e.NoSuchContainer:
cont = None
# Container not found, create it
if cont is None:
try:
print ("INFO: Container '%s' not found, creating with TTL set "
"to %d..." % (args.container, args.cdn_ttl))
cont = cf.create_container(args.container)
cont.make_public(ttl=args.cdn_ttl)
except e.CDNFailed:
print "ERROR: Could not CDN enable the container"
sys.exit(9)
except:
print "ERROR: Could not create container", args.container
sys.exit(10)
# Otherwise inform the user/client that the directory exists and
# determine if we can proceed (is the force flag set)
else:
print ("INFO: Container '%s' found with TTL set to %d"
% (cont.name, cont.cdn_ttl))
if args.force:
print "INFO: Proceeding as force flag is set"
else:
print "INFO: Force flag not set, exiting..."
sys.exit(11)
# Set the static web index file (metadata/header)
print "INFO: Setting static web index file to", args.index
meta = {"X-Container-Meta-Web-Index": args.index}
cf.set_container_metadata(cont, meta)
# Check and confirm if the index file exists, otherwise create a
# placeholder and inform/warn the client of the action
index_file = args.directory + "/" + args.index
if not os.path.isfile(index_file):
print ("INFO: Index file '%s' not found, creating a placeholder"
% (index_file))
f = open(index_file,'w')
f.write("Index page placeholder\n")
f.close()
# Start the upload
print "INFO: Beginning directory/folder upload"
(upload_key, total_bytes) = cf.upload_folder(args.directory, cont)
# Inform the user of the total upload size
print ("INFO: Total upload size: %d bytes (%.2f MB)"
% (total_bytes, total_bytes / 1024.0 / 1024))
# Prepare progress toolbar
sys.stdout.write("[%s]" % (" " * TOOLBAR_WIDTH))
sys.stdout.flush()
# Return to start of line, after '['
sys.stdout.write("\b" * (TOOLBAR_WIDTH + 1))
# Print the upload progress (1 second interval)
uploaded = 0
while uploaded < total_bytes:
uploaded = cf.get_uploaded(upload_key)
progress = int(ceil((uploaded * 100.0) / total_bytes / 2))
sys.stdout.write("%s" % ("=" * progress))
sys.stdout.flush()
sys.stdout.write("\b" * (progress)) # Reset progress bar
sleep(1)
print
# Upload completed, print object count and CDN URIs
objs = cf.get_container_object_names(cont)
print "INFO: Number of objects uploaded: %d" % (len(objs))
# Attempt to create the new CNAME record
cname_rec = {"type": "CNAME",
"name": args.fqdn,
"data": cont.cdn_uri.replace("http://", ""),
"ttl": args.cname_ttl}
try:
rec = zone.add_record(cname_rec)
print "INFO: DNS record successfully added"
print ("-- Record details\n\tName: %s\n\tType: %s\n\tIP address: "
"%s\n\tTTL: %s") % (rec[0].name, rec[0].type, rec[0].data,
rec[0].ttl)
print "INFO: All requests completed successfully"
except e.DomainRecordAdditionFailed as err:
print "ERROR: Record addition request failed\nReason:", err
sys.exit(12)
def resize_project_avatars():
"""Resize project avatars to 512px."""
if app.config['UPLOAD_METHOD'] == 'rackspace':
import pyrax
import tempfile
import requests
from PIL import Image
import time
import pybossa.cache.projects as cached_apps
# Disable cache to update the data in it :-)
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
api_key=app.config['RACKSPACE_API_KEY'],
region=app.config['RACKSPACE_REGION'])
cf = pyrax.cloudfiles
#apps = Project.query.all()
file_name = 'project_id_updated_thumbnails.txt'
project_id_updated_thumbnails = []
if os.path.isfile(file_name):
f = open(file_name, 'r')
project_id_updated_thumbnails = f.readlines()
f.close()
apps = Project.query.filter(~Project.id.in_(project_id_updated_thumbnails)).all()
#apps = [Project.query.get(2042)]
print "Downloading avatars for %s projects" % len(apps)
dirpath = tempfile.mkdtemp()
f = open(file_name, 'a')
for a in apps:
try:
if a.info.get('container'):
cont = cf.get_container(a.info['container'])
avatar_url = "%s/%s" % (cont.cdn_ssl_uri, a.info['thumbnail'])
r = requests.get(avatar_url, stream=True)
if r.status_code == 200:
print "Downloading avatar for %s ..." % a.short_name
prefix = time.time()
filename = "app_%s_thumbnail_%s.png" % (a.id, prefix)
with open(os.path.join(dirpath, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
# Resize image
im = Image.open(os.path.join(dirpath, filename))
size = 512, 512
tmp = im.resize(size, Image.ANTIALIAS)
scale_down_img = tmp.convert('P', colors=255, palette=Image.ADAPTIVE)
scale_down_img.save(os.path.join(dirpath, filename), format='png')
print "New scaled down image created!"
print "%s" % (os.path.join(dirpath, filename))
print "---"
chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
filename))
cf.upload_file(cont,
os.path.join(dirpath, filename),
obj_name=filename,
etag=chksum)
old_avatar = a.info['thumbnail']
# Update new values
a.info['thumbnail'] = filename
#a.info['container'] = "user_%s" % u.id
db.session.commit()
f = open(file_name, 'a')
f.write("%s\n" % a.id)
# delete old avatar
obj = cont.get_object(old_avatar)
obj.delete()
print "Done!"
cached_apps.get_app(a.short_name)
else:
print "No Avatar found."
else:
print "No avatar found."
except pyrax.exceptions.NoSuchObject:
print "Previous avatar not found, so not deleting it."
except:
raise
print "No Avatar, this project will use the placehoder."
f.close()
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import time
import pyrax
import pyrax.exceptions as exc
import pyrax.utils as utils
pyrax.set_setting("identity_type", "rackspace")
creds_file = os.path.expanduser("~/.rackspace_cloud_credentials")
pyrax.set_credential_file(creds_file)
cf = pyrax.cloudfiles
cont_name = pyrax.utils.random_name(8)
cont = cf.create_container(cont_name)
# pyrax has a utility for creating temporary local directories that clean
# themselves up.
with utils.SelfDeletingTempDirectory() as tmpfolder:
# Create a bunch of files
for idx in xrange(13):
fname = "file_%s" % idx
pth = os.path.join(tmpfolder, fname)
with open(pth, "w") as tmp:
import urllib
'''
Example Env vars:
export OS_USERNAME=YOUR_USERNAME
export OS_REGION=LON
export OS_API_KEY=fc8234234205234242ad8f4723426cfe
'''
# Consume our environment vars
app_name = os.environ.get('NAMESPACE', 'win')
environment_name = os.environ.get('ENVIRONMENT', 'stg')
domain_name = os.environ.get('DOMAIN_NAME', None)
# Authenticate
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_setting("region", os.environ.get('OS_REGION', "LON"))
pyrax.set_credentials(os.environ.get('OS_USERNAME'),
os.environ.get('OS_API_KEY'))
# Set up some aliases
clb = pyrax.cloud_loadbalancers
au = pyrax.autoscale
dns = pyrax.cloud_dns
# Derived names
asg_name = app_name + "-" + environment_name
lb_name = asg_name + '-lb'
subdomain_name = asg_name
# Try to find the ASG by naming convention.
def main():
# check '~/.pyraxshell' and config files exist, create them if missing
if not check_dir_home():
print("This is the first time 'pyraxshell' runs, please, configure "
"'%s' according to your needs" % CONFIG_FILE)
#create db
DB()
Sessions.Instance().create_table_sessions() # @UndefinedVariable
Sessions.Instance().create_table_commands() # @UndefinedVariable
# create default configuration file
Configuration.Instance() # @UndefinedVariable
sys.exit(0)
# ########################################
# VERSION CHECK
if not version.check_version_file():
sys.exit(1)
# ########################################
# LOGGING
start_logging()
logging.debug('starting')
# from baseconfigfile import BaseConfigFile
# bcf = BaseConfigFile()
# ########################################
# ACCOUNTS
accounts = Account.Instance() # @UnusedVariable @UndefinedVariable
# config file is read by 'BaseConfigFile' constructor
# ########################################
# CONFIGURATION
cfg = Configuration.Instance() # @UndefinedVariable
# override settings with CLI params
cfg.parse_cli(sys.argv)
logging.debug("configuration: %s" % cfg)
# set user's log level if specified
if not Configuration.Instance().log_level == None: # @UndefinedVariable
l = logging.getLogger()
for h in l.handlers:
h.setLevel(cfg.log_level)
# ########################################
# START SESSION
Sessions.Instance().start_session() # @UndefinedVariable
# Sessions.Instance().insert_table_commands('IN', 'OUT') # @UndefinedVariable
# ########################################
# DO STUFF
# handle configuration
if cfg.pyrax_http_debug == True:
pyrax.set_http_debug(True)
if cfg.pyrax_no_verify_ssl == True:
# see: https://github.com/rackspace/pyrax/issues/187
pyrax.set_setting("verify_ssl", False)
# start notifier
Notifier().start()
# main loop
Cmd_Pyraxshell().cmdloop()
def setup():
default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
# pyrax does not honor the environment variable CLOUD_VERIFY_SSL=False, so let's help pyrax
if 'CLOUD_VERIFY_SSL' in os.environ:
pyrax.set_setting(
'verify_ssl', os.environ['CLOUD_VERIFY_SSL']
in [1, 'true', 'True'])
env = get_config(p, 'rax', 'environment', 'RAX_ENV', None)
if env:
pyrax.set_environment(env)
keyring_username = pyrax.get_setting('keyring_username')
# Attempt to grab credentials from environment first
creds_file = get_config(p, 'rax', 'creds_file', 'RAX_CREDS_FILE', None)
if creds_file is not None:
creds_file = os.path.expanduser(creds_file)
else:
# But if that fails, use the default location of
# ~/.rackspace_cloud_credentials
if os.path.isfile(default_creds_file):
creds_file = default_creds_file
elif not keyring_username:
sys.stderr.write('No value in environment variable %s and/or no '
'credentials file at %s\n' %
('RAX_CREDS_FILE', default_creds_file))
sys.exit(1)
identity_type = pyrax.get_setting('identity_type')
pyrax.set_setting('identity_type', identity_type or 'rackspace')
region = pyrax.get_setting('region')
try:
if keyring_username:
pyrax.keyring_auth(keyring_username, region=region)
else:
pyrax.set_credential_file(creds_file, region=region)
except Exception as e:
sys.stderr.write("%s: %s\n" % (e, e.message))
sys.exit(1)
regions = []
if region:
regions.append(region)
else:
region_list = get_config(p,
'rax',
'regions',
'RAX_REGION',
'all',
islist=True)
for region in region_list:
region = region.strip().upper()
if region == 'ALL':
regions = pyrax.regions
break
elif region not in pyrax.regions:
sys.stderr.write('Unsupported region %s' % region)
sys.exit(1)
elif region not in regions:
regions.append(region)
return regions
def bootstrap_avatars():
"""Download current links from user avatar and projects to real images hosted in the
PyBossa server."""
import requests
import os
import time
from urlparse import urlparse
from PIL import Image
def get_gravatar_url(email, size):
# import code for encoding urls and generating md5 hashes
import urllib, hashlib
# construct the url
gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
return gravatar_url
with app.app_context():
if app.config['UPLOAD_METHOD'] == 'local':
users = User.query.order_by('id').all()
print "Downloading avatars for %s users" % len(users)
for u in users:
print "Downloading avatar for %s ..." % u.name
container = "user_%s" % u.id
path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
try:
print get_gravatar_url(u.email_addr, 100)
r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
if r.status_code == 200:
if not os.path.isdir(path):
os.makedirs(path)
prefix = time.time()
filename = "%s_avatar.png" % prefix
with open(os.path.join(path, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
u.info['avatar'] = filename
u.info['container'] = container
db.session.commit()
print "Done!"
else:
print "No Gravatar, this user will use the placeholder."
except:
raise
print "No gravatar, this user will use the placehoder."
apps = Project.query.all()
print "Downloading avatars for %s projects" % len(apps)
for a in apps:
if a.info.get('thumbnail') and not a.info.get('container'):
print "Working on project: %s ..." % a.short_name
print "Saving avatar: %s ..." % a.info.get('thumbnail')
url = urlparse(a.info.get('thumbnail'))
if url.scheme and url.netloc:
container = "user_%s" % a.owner_id
path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
try:
r = requests.get(a.info.get('thumbnail'), stream=True)
if r.status_code == 200:
prefix = time.time()
filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
if not os.path.isdir(path):
os.makedirs(path)
with open(os.path.join(path, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
a.info['thumbnail'] = filename
a.info['container'] = container
db.session.commit()
print "Done!"
except:
print "Something failed, this project will use the placehoder."
if app.config['UPLOAD_METHOD'] == 'rackspace':
import pyrax
import tempfile
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
api_key=app.config['RACKSPACE_API_KEY'],
region=app.config['RACKSPACE_REGION'])
cf = pyrax.cloudfiles
users = User.query.all()
print "Downloading avatars for %s users" % len(users)
dirpath = tempfile.mkdtemp()
for u in users:
try:
r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
if r.status_code == 200:
print "Downloading avatar for %s ..." % u.name
container = "user_%s" % u.id
try:
cf.get_container(container)
except pyrax.exceptions.NoSuchContainer:
cf.create_container(container)
cf.make_container_public(container)
prefix = time.time()
filename = "%s_avatar.png" % prefix
with open(os.path.join(dirpath, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
filename))
cf.upload_file(container,
os.path.join(dirpath, filename),
obj_name=filename,
etag=chksum)
u.info['avatar'] = filename
u.info['container'] = container
db.session.commit()
print "Done!"
else:
print "No Gravatar, this user will use the placeholder."
except:
print "No gravatar, this user will use the placehoder."
apps = Project.query.all()
print "Downloading avatars for %s projects" % len(apps)
for a in apps:
if a.info.get('thumbnail') and not a.info.get('container'):
print "Working on project: %s ..." % a.short_name
print "Saving avatar: %s ..." % a.info.get('thumbnail')
url = urlparse(a.info.get('thumbnail'))
if url.scheme and url.netloc:
container = "user_%s" % a.owner_id
try:
cf.get_container(container)
except pyrax.exceptions.NoSuchContainer:
cf.create_container(container)
cf.make_container_public(container)
try:
r = requests.get(a.info.get('thumbnail'), stream=True)
if r.status_code == 200:
prefix = time.time()
filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
with open(os.path.join(dirpath, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
filename))
cf.upload_file(container,
os.path.join(dirpath, filename),
obj_name=filename,
etag=chksum)
a.info['thumbnail'] = filename
a.info['container'] = container
db.session.commit()
print "Done!"
except:
print "Something failed, this project will use the placehoder."
def _create_connection(self):
username, api_key = self.credentials_manager.get_credentials()
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(username, api_key)
return pyrax.connect_to_cloudfiles()
def resize_avatars():
"""Resize avatars to 512px."""
if app.config['UPLOAD_METHOD'] == 'rackspace':
import pyrax
import tempfile
import requests
from PIL import Image
import time
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
api_key=app.config['RACKSPACE_API_KEY'],
region=app.config['RACKSPACE_REGION'])
cf = pyrax.cloudfiles
user_id_updated_avatars = []
if os.path.isfile('user_id_updated_avatars.txt'):
t = open('user_id_updated_avatars.txt', 'r')
user_id_updated_avatars = t.readlines()
t.close()
users = User.query.filter(~User.id.in_(user_id_updated_avatars)).all()
print "Downloading avatars for %s users" % len(users)
dirpath = tempfile.mkdtemp()
f = open('user_id_updated_avatars.txt', 'a')
for u in users:
try:
if u.info.get('container'):
cont = cf.get_container(u.info['container'])
if cont.cdn_ssl_uri:
avatar_url = "%s/%s" % (cont.cdn_ssl_uri, u.info['avatar'])
else:
cont.make_public()
avatar_url = "%s/%s" % (cont.cdn_ssl_uri, u.info['avatar'])
r = requests.get(avatar_url, stream=True)
if r.status_code == 200:
print "Downloading avatar for %s ..." % u.name
#container = "user_%s" % u.id
#try:
# cf.get_container(container)
#except pyrax.exceptions.NoSuchContainer:
# cf.create_container(container)
# cf.make_container_public(container)
prefix = time.time()
filename = "%s_avatar.png" % prefix
with open(os.path.join(dirpath, filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
# Resize image
im = Image.open(os.path.join(dirpath, filename))
size = 512, 512
tmp = im.resize(size, Image.ANTIALIAS)
scale_down_img = tmp.convert('P', colors=255, palette=Image.ADAPTIVE)
scale_down_img.save(os.path.join(dirpath, filename), format='png')
print "New scaled down image created!"
print "%s" % (os.path.join(dirpath, filename))
print "---"
chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
filename))
cf.upload_file(cont,
os.path.join(dirpath, filename),
obj_name=filename,
etag=chksum)
old_avatar = u.info['avatar']
# Update new values
u.info['avatar'] = filename
u.info['container'] = "user_%s" % u.id
db.session.commit()
# Save the user.id to avoid downloading it again.
f = open('user_id_updated_avatars.txt', 'a')
f.write("%s\n" % u.id)
# delete old avatar
obj = cont.get_object(old_avatar)
obj.delete()
print "Done!"
else:
print "No Avatar found."
else:
f.write("%s\n" % u.id)
print "No avatar found"
except pyrax.exceptions.NoSuchObject:
print "Previous avatar not found, so not deleting it."
except:
raise
print "No Avatar, this user will use the placehoder."
f.close()
def main():
parser = argparse.ArgumentParser(
description="Migrate images from rackspace to s3")
parser.add_argument("-a",
"--action",
required=True,
help="""
Action to perform
- fetch: fetching all the filenames
on rackspace and store it to a file
- setup: put filenames from fetch
into redis for running
- migrate: download file from rackspace
and upload to s3
""")
parser.add_argument('-n',
'--num_workers',
required=False,
type=int,
default=multiprocessing.cpu_count(),
help='Number of workers. Default is number of cpus')
args = parser.parse_args()
if not S3_ACCESS_KEY or not S3_SECRET_KEY:
print("""Please set up environment variable
S3_ACCESS_KEY and S3_SECRET_KEY""")
return
if (not RACKSPACE_IDENTITY_TYPE or not RACKSPACE_USERNAME
or not RACKSPACE_API_KEY):
print(""" Please set up environment variable
RACKSPACE_IDENTITY_TYPE
RACKSPACE_USERNAME
RACKSPACE_API_KEY
""")
return
if not s3_bucket or not rackspace_container:
print("Please setup s3_bucket and rackspace_container")
return
redis_host, redis_port = redis_url.split(':')
redis_connection = redis.StrictRedis(host=redis_host,
port=int(redis_port),
db=redis_db)
if args.action == 'migrate':
pyrax.set_setting('identity_type', RACKSPACE_IDENTITY_TYPE)
pyrax.set_credentials(RACKSPACE_USERNAME, RACKSPACE_API_KEY)
container = pyrax.cloudfiles.get_container(rackspace_container)
run_s3(container, redis_connection, args.num_workers)
elif args.action == 'fetch':
pyrax.set_setting('identity_type', RACKSPACE_IDENTITY_TYPE)
pyrax.set_credentials(RACKSPACE_USERNAME, RACKSPACE_API_KEY)
container = pyrax.cloudfiles.get_container(rackspace_container)
fetch_filenames(container, id_filename)
elif args.action == 'setup':
total = redis_connection.llen(redis_key)
if total > 0:
raise Exception('Not empty')
setup_filenames(redis_connection, id_filename)
def __init__(self, filename, region, container_name, use_public):
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_credential_file(filename)
self.cf = pyrax.connect_to_cloudfiles(region=region, public=use_public)
self.cnt = self.cf.create_container(container_name)
def main():
# Set default metadata key to search for
METAKEY = "MyGroup0_clb"
# Set default metadata value to search for
METAVALUE = "clb0"
# Set default location of pyrax configuration file
CREDFILE = "~/.rackspace_cloud_credentials"
# Set the default location of log files
LOGPATH = "/var/log/"
# A short description
parser = argparse.ArgumentParser(
description=("Automatically update load balancer nodes"))
# Set argument for the region
parser.add_argument("-r",
"--region",
action="store",
required=False,
metavar="REGION",
type=str,
help=("Region where servers should be built (defaults"
" to 'LON'"),
choices=["ORD", "DFW", "LON"
"IAD", "SYD"],
default="LON")
# Set argument for the metadata key
parser.add_argument("-mk",
"--metakey",
action="store",
required=False,
metavar="META_KEY",
type=str,
help=("Matadata key that desired node has"),
default=METAKEY)
# Set argument for the metadata value
parser.add_argument("-mv",
"--metavalue",
action="store",
required=False,
metavar="META_VALUE",
type=str,
help=("Metadata value that desired node has"),
default=METAVALUE)
# Set argument for the cloud load balancer ID
parser.add_argument("-i",
"--clbid",
action="store",
required=True,
metavar="CLB_ID",
type=int,
help=("Cloud Load Balancer ID"))
# Set argument for the pyrax credentials file loacation
parser.add_argument("-c",
"--credfile",
action="store",
required=False,
metavar="CREDENTIALS_FILE",
type=str,
help=("The location of your pyrax configuration file"),
default=CREDFILE)
# Set argument for the log file directory
parser.add_argument("-p",
"--logpath",
action="store",
required=False,
metavar="LOG_DIRECTORY",
type=str,
help=("The directory to create log files in"),
default=LOGPATH)
# Set argument for verbosity
parser.add_argument("-v",
"--verbose",
action="store_true",
required=False,
help=("Turn on debug verbosity"),
default=False)
# Parse the arguments
args = parser.parse_args()
# Configure logging
logFormatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
rootLogger = logging.getLogger()
# Check what level we should log with
if args.verbose:
rootLogger.setLevel(logging.DEBUG)
else:
rootLogger.setLevel(logging.WARNING)
# Configure logging to console
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
# Configure logging to file
try:
fileHandler = logging.FileHandler("{0}/{1}.log".format(
args.logpath, os.path.basename(__file__)))
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
except IOError:
rootLogger.critical("Unable to write to log file directory '%s'" %
(logpath))
exit(1)
# Define the authentication credentials file location and request that
# pyrax makes use of it. If not found, let the client/user know about it.
# Use a credential file in the following format:
# [rackspace_cloud]
# username = myusername
# api_key = 01234567890abcdef
# region = LON
pyrax.set_setting("identity_type", "rackspace")
# Test that the pyrax configuration file provided exists
try:
creds_file = os.path.expanduser(args.credfile)
pyrax.set_credential_file(creds_file, args.region)
# Exit if authentication fails
except e.AuthenticationFailed:
rootLogger.critical(
"Authentication failed. Please check and confirm"
"that the API username, key, and region are in place"
"and correct.")
exit(1)
# Exit if file does not exist
except e.FileNotFound:
rootLogger.critical("Credentials file '%s' not found" % (creds_file))
rootLogger.info(
"%s", """Use a credential file in the following format:\n
[rackspace_cloud]
username = myuseername
api_key = 01sdf444g3ffgskskeoek0349
""")
exit(2)
# Shorten some pyrax calls
cs = pyrax.cloudservers
clb = pyrax.cloud_loadbalancers
# Check that we have some servers in this region
if not cs.servers.list():
rootLogger.critical("No servers found in region '%s'" % (args.region))
exit(3)
# Check that we have the load balancer in this region
if not clb.find(id=args.clbid):
rootLogger.critical("No matching load balancer found in region '%s'" %
(args.region))
exit(4)
# Find our load balancer
myclb = clb.find(id=args.clbid)
# Let the user know we found the load balancer
rootLogger.info('Found Load Balancer, id: %i status: %s' %
(myclb.id, myclb.status))
# Create some empty lists
clbips = []
csips = []
# Make a list of nodes currently in out load balancer.
for node in myclb.nodes:
if node.condition == 'ENABLED' or 'DISABLED':
clbips.append(node.address)
# Let the user know what nodes we found
rootLogger.info('Current load balancer nodes: %s' % (clbips))
# Make a list of servers that match out metadata
for server in cs.servers.list():
# filter out only ACTIVE ones
if server.status == 'ACTIVE':
# Check for servers that match both your meta key and value
if args.metakey in server.metadata and server.metadata[
args.metakey] == args.metavalue:
# Grab the private ip address of matching server
csips.append(server.networks['private'][0])
# Let the user know what servers match our metadata
rootLogger.info('Cloud servers matching metadata: %s' % (csips))
if set(clbips) == set(csips):
rootLogger.info("No update required")
exit(0)
else:
pass
# Make a list of new ip's to add into load balancer
newips = [x for x in csips if x not in clbips]
# Make a list of old ip's to remove from load balancer
oldips = [x for x in clbips if x not in csips]
# Check out verbosity
if args.verbose:
v = True
else:
v = False
# Set our load balancing port
myport = myclb.port
# If we have new ip's then add them
if newips:
# Let the user know what IP's we are adding
rootLogger.warning('Nodes to add to load balancer %i: %s' %
(myclb.id, newips))
for ip in newips:
new_node = clb.Node(address=ip, port=myport, condition="ENABLED")
myclb.add_nodes([new_node])
# Wait for the load balancer to update
pyrax.utils.wait_until(myclb,
"status",
"ACTIVE",
interval=1,
attempts=30,
verbose=v)
# If we have old ip'd then remove them
if oldips:
# Let the user know what ips' we are removing
rootLogger.warning('Nodes to remove from load balancer %i: %s' %
(myclb.id, oldips))
for node in myclb.nodes:
if node.address in oldips:
node.delete()
pyrax.utils.wait_until(myclb,
"status",
"ACTIVE",
interval=1,
attempts=30,
verbose=v)
# Let the user know we are done
rootLogger.info("Update complete")
exit(0)
# Attempt to grab credentials from environment first
try:
creds_file = os.path.expanduser(os.environ['RAX_CREDS_FILE'])
except KeyError, e:
# But if that fails, use the default location of
# ~/.rackspace_cloud_credentials
if os.path.isfile(default_creds_file):
creds_file = default_creds_file
elif not keyring_username:
sys.stderr.write('No value in environment variable %s and/or no '
'credentials file at %s\n' %
(e.message, default_creds_file))
sys.exit(1)
identity_type = pyrax.get_setting('identity_type')
pyrax.set_setting('identity_type', identity_type or 'rackspace')
region = pyrax.get_setting('region')
try:
if keyring_username:
pyrax.keyring_auth(keyring_username, region=region)
else:
pyrax.set_credential_file(creds_file, region=region)
except Exception, e:
sys.stderr.write("%s: %s\n" % (e, e.message))
sys.exit(1)
regions = []
if region:
regions.append(region)
def init_pyrax(self, region):
self.region = region
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(self.username, self.api_key)
self.cs = pyrax.connect_to_cloudservers(region, verify_ssl=True)
def wrapper(*args, **kwargs):
pyrax.set_setting('identity_type', app.config['AUTH_SYSTEM'])
pyrax.set_default_region(app.config['AUTH_SYSTEM_REGION '])
pyrax.set_credentials(app.config['USERNAME'], app.config['PASSWORD'])
nova = pyrax.cloudservers
return func(*args, **kwargs)
def main():
# Define the authentication credentials file location and request that
# pyrax makes use of it. If not found, let the client/user know about it.
# Use a credential file in the following format:
# [rackspace_cloud]
# username = myusername
# api_key = 01234567890abcdef
# region = LON
# Set identity type as rackspace
pyrax.set_setting("identity_type", "rackspace")
# Test that the pyrax configuration file provided exists
try:
creds_file = os.path.expanduser(args.credfile)
pyrax.set_credential_file(creds_file, args.region)
# Exit if authentication fails
except pex.AuthenticationFailed:
rootLogger.critical("Authentication failed")
rootLogger.info(
"%s", """Please check and confirm that the API username,
key, and region are in place and correct."""
)
exit(2)
# Exit if file does not exist
except pex.FileNotFound:
rootLogger.critical("Credentials file '%s' not found" % (creds_file))
rootLogger.info(
"%s", """Use a credential file in the following format:\n
[rackspace_cloud]
username = myuseername
api_key = 01sdf444g3ffgskskeoek0349""")
exit(3)
# Shorten the call to cloud databases
cdb = pyrax.cloud_databases
# Try to find the instance matching UUID provided
try:
mycdb = cdb.find(id=args.instance)
except pex.NotFound:
rootLogger.critical("No instances found matching '%s'" %
(args.instance))
exit(4)
# Create a new backup
if args.description:
description = args.description
else:
description = "Created on " + datetime.datetime.now().strftime(
"%Y-%b-%d-%H:%M")
try:
rootLogger.info("Creating backup of '%s'." % (mycdb.name))
new_backup = mycdb.create_backup(
args.backup + '-' + datetime.datetime.now().strftime("%y%m%d%H%M"),
description=description)
except pex.ClientException:
type, value, traceback = sys.exc_info()
rootLogger.critical(value.message)
exit(5)
# Put our current backups in a list
backups = []
for backup in mycdb.list_backups():
if backup.name.startswith(args.backup) and backup.name[10:].isnumeric:
backups.append(backup)
# Sort out backups by date (this is probably not a very reliable way)
backups.sort(key=lambda backup: backup.created, reverse=True)
# Print our current backups
rootLogger.info("Current backups of '%s' below:" % (mycdb.name))
for backup in backups:
rootLogger.info("Name: '%s' Created on '%s'" %
(backup.name, backup.created))
# Check if backups need to be deleted
if len(backups) > args.number and args.number is not 0:
# Warn of backups being deleted
rootLogger.warning("There are '%i' backups. Need to delete '%i'." %
(len(backups), len(backups) - args.number))
# Delete oldest backups if current backups > target backups
for backup in backups[args.number:]:
try:
rootLogger.warning("The below backups will be deleted.")
rootLogger.warning("Name: '%s' Created on '%s'." %
(backup.name, backup.created))
backup.delete()
except pex.ClientException:
type, value, traceback = sys.exc_info()
rootLogger.critical(value.message)
exit(6)
# Wait until the instance is active
pyrax.utils.wait_for_build(mycdb,
"status",
"ACTIVE",
interval=1,
attempts=30)
exit(0)
server_name = a
else:
if o in ("-d", "--debug"):
debug_flag = True
else:
if o in ("--remove-after"):
remove_after = True
else:
assert False, "unhandled option"
server_log_id = "Node:" + server_name
utils.log_msg(" ".join([server_log_id, "Starting build-one"]), "INFO",
config.print_to)
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_setting("region", config.cloud_region)
try:
pyrax.set_credentials(config.cloud_user,
config.cloud_api_key,
region=config.cloud_region)
except pyrax.exc.AuthenticationFailed:
utils.log_msg(
" ".join([server_log_id, "Pyrax auth failed using",
config.cloud_user]), "ERROR", config.print_to)
utils.log_msg(
" ".join([server_log_id, "Authenticated using", config.cloud_user]),
"INFO", config.print_to)
cs = pyrax.cloudservers
#Authentication
import os, os.path, pyrax, pyrax.exceptions as exc, time
pyrax.set_setting("identity_type", "")
pyrax.set_default_region('')
pyrax.set_credentials("", "")
cf = pyrax.cloudfiles
#Container & StorageObject
cont = cf.get_container("")
objs = cont.get_objects()
#Fetch File
for obj in objs:
cf.download_object("", obj.name, "", structure=False)
import pyrax
import pyrax.exceptions
from pasteraw import app
from pasteraw import log
ENABLED = False
if app.config['CLOUD_ID_TYPE'] == 'rackspace':
try:
pyrax.set_setting('identity_type', app.config['CLOUD_ID_TYPE'])
log.info('Setting region', region=app.config['CLOUD_REGION'])
pyrax.set_setting('region', app.config['CLOUD_REGION'])
pyrax.set_setting('use_servicenet', True)
log.info('Logging into Rackspace',
username=app.config['RACKSPACE_USERNAME'],
api_key=bool(app.config['RACKSPACE_API_KEY']))
pyrax.set_credentials(app.config['RACKSPACE_USERNAME'],
app.config['RACKSPACE_API_KEY'])
ENABLED = True
except (pyrax.exceptions.PyraxException, AttributeError) as e:
log.warning('Unable to authenticate using pyrax', exception=e)
elif app.config['CLOUD_ID_TYPE'] == 'keystone':
raise NotImplementedError(
'pyrax does not document how to provide keystone credentials '
'"directly".')
else:
log.warning(
# django/tastypie/libs
import envitro
import pyrax
import warnings
from django.conf import settings
# pyrax connection
RACKSPACE_USERNAME = envitro.str('RACKSPACE_USERNAME')
RACKSPACE_APIKEY = envitro.str('RACKSPACE_APIKEY')
# attributes
try:
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_setting('region', 'DFW')
pyrax.set_credentials(RACKSPACE_USERNAME, RACKSPACE_APIKEY)
cloud_files = pyrax.connect_to_cloudfiles(
public=settings.CLOUDFILES_PUBLIC_NETWORK)
except:
warnings.warn('Not able to setup CloudFiles connection')
cloud_files = None
def __init__(self, **kwargs):
super(SwiftNotebookManager, self).__init__(**kwargs)
pyrax.set_setting("custom_user_agent", self.user_agent)
def cloudfiles_func(settings, filename, results):
"""
Uploads files to Rackspace Cloud Files.
"""
name = threading.currentThread().getName()
logger = logging.getLogger(__name__ + "." + name)
creds_file = settings['credential_file']
pyrax.set_credential_file(creds_file)
pyrax.set_setting('use_servicenet', settings['use_snet'])
region = settings['region']
container_name = settings['container_name']
nest_by_timestamp = settings.get('nest_by_timestamp', False)
obj_ttl = settings.get('set_ttl', None)
try:
cf = pyrax.connect_to_cloudfiles(region=region)
container = cf.get_container(container_name)
except:
logger.error(
"Unable to connect to cloudfiles. Transfer for {0} aborted, failing gracefully."
.format(filename))
results.append(name)
return
if os.path.getsize(filename) >= 5368709120:
logger.error(
"{0} is too large. Files over 5GB are not currently supported.".
format(filename))
results.append(name)
return
obj_name = os.path.basename(filename)
# Create new obj_name for nested directory
if nest_by_timestamp:
t = os.path.getmtime(filename)
d = dt.fromtimestamp(t)
obj_name = "{year}/{month}/{day}/{filename}".format(
year=d.strftime("%Y"),
month=d.strftime("%m"),
day=d.strftime("%d"),
filename=obj_name)
chksum = pyrax.utils.get_checksum(filename)
for i in range(MAX_RETRIES):
try:
start = time.time()
#Used for testing the retry
#raise pyrax.exceptions.UploadFailed()
obj = container.upload_file(filename,
obj_name=obj_name,
etag=chksum,
ttl=obj_ttl)
end = time.time()
logger.debug("%s transferred to %s in %.2f secs." %
(filename, container_name, (end - start)))
break
except pyrax.exceptions.UploadFailed:
logger.warning("Upload to container:%s in %s failed, retry %d" %
(container_name, region, i + 1))
time.sleep(2)
else:
logger.error("Upload to container:%s in %s failed!" %
(container_name, region))
results.append(name)
def cache_clean(folder, extension):
# NOTE: Manually install gevent & pyrax, no need for it to be depenency just for this method.
from gevent import monkey
from gevent.pool import Pool
from gevent import Timeout
monkey.patch_all()
import six
import pyrax
import logging
from mfr.server import settings
# Monkey patch pyrax for python 3 compatibility.
def _add_details(self, info):
"""
Takes the dict returned by the API call and sets the
corresponding attributes on the object.
"""
for (key, val) in six.iteritems(info):
if six.PY2 and isinstance(key, six.text_type):
key = key.encode(pyrax.get_encoding())
elif isinstance(key, bytes):
key = key.decode("utf-8")
setattr(self, key, val)
pyrax.resource.BaseResource._add_details = _add_details
# WARNING: We are using provider specific functions to enumerate files to quickly
# purge the cache, which can contain hundreds of thousands of objects. Thus
# asserting the provider, we will need to update if we move providers.
assert settings.CACHE_PROVIDER_NAME == 'cloudfiles'
logging.captureWarnings(True)
pyrax.set_setting('identity_type', 'rackspace')
pyrax.set_setting('verify_ssl', True)
pyrax.set_credentials(settings.CACHE_PROVIDER_CREDENTIALS['username'],
settings.CACHE_PROVIDER_CREDENTIALS['token'])
cf = pyrax.connect_to_cloudfiles(
region=settings.CACHE_PROVIDER_CREDENTIALS['region'].upper(),
public=True)
container = cf.get_container(settings.CACHE_PROVIDER_SETTINGS['container'])
def delete_object(obj):
# added timeout of 5 seconds just in case
with Timeout(5, False):
try:
print(obj)
obj.delete()
except Exception as ex:
print(ex)
pool = Pool(100)
objects = container.get_objects(prefix=folder, limit=5000, marker='')
while objects:
for obj in objects:
if obj.name.endswith(extension):
pool.spawn(delete_object, obj)
objects = container.get_objects(prefix=folder,
limit=5000,
marker=objects[-1].name)
pool.join()
def enable_public_cloud(self, username, api_key):
import pyrax
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credentials(username, api_key)
self.cloudservers = pyrax.cloudservers
print "Run setup_rackspace_api_client.sh"
exit()
import os
import sys
if len(sys.argv) < 3:
print "Usage: ./%s server-name command" % __file__
exit()
server_name = sys.argv[1]
command = sys.argv[2]
creds_file = os.path.expanduser("~/.rackspace_cloud_credentials")
pyrax.set_setting("region", "LON")
pyrax.set_setting("identity_type", "rackspace")
pyrax.set_credential_file(creds_file)
cs = pyrax.cloudservers
server = cs.servers.find(name=server_name)
interface_manager = VirtualInterfaceManager(cs)
if command == "delete":
interfaces = interface_manager.list(server.id)
interface_id_map = dict(
(interface.ip_addresses[0].get('network_label'), interface.id)
for interface in interfaces)
print "Deleting %s interface %s from server %s " % (
def upload(self):
print "Uploading file to RackSpace CDN..."
pyrax.set_setting("region", self.region)
cf = auth(self.user)
if self.verbose: print("getting container...")
if self.debug_mode:
print("cf.get_all_containers", cf.get_all_containers())
container = cf.get_container(self.bucket_id)
# check if object already exists:
# if same name and same md5, don't bother re-uploading.
try:
if self.verbose: print("trying to get existing object...")
obj = container.get_object(self.key_id)
if self.verbose: print("checking for already_there...")
already_there = obj.etag == pyrax.utils.get_checksum(
self.pathname,)
ret = True
except pyrax.exceptions.NoSuchObject as e:
already_there = False
if not already_there:
done=False
while not done:
pf = ProgressFile(self.pathname, 'r')
try:
# actually upload
if self.verbose: print("container.upload_file...")
obj = container.upload_file(pf, obj_name = self.key_id)
if self.debug_mode:
import code
code.interact(local=locals())
done = True
ret = True
except pyrax.exceptions.ClientException as e:
print "caught pyrax.exceptions.ClientException as e"
print e
print e.code, e.details, e.message
if e.code in [408,503]:
# 408 Request timeout
# 503 Service Unavailable - The server is currently unavailable. Please try again at a later time.
print "looping..."
continue
print e
# self.ret_text = "rax error: %s" % ( e.body )
import code
code.interact(local=locals())
except Exception as e:
print "caught Exception as e"
"""
HTTPSConnectionPool(host='storage101.ord1.clouddrive.com', port=443): Max retries exceeded with url: /v1/MossoCloudFS_fd6d6695-7fe7-4f77-9b4a-da7696e71dc2/fosdem/veyepar/debian/debconf14/dv/plenary/2014-08-23/16_00_03.ogv (Caused by <class 'socket.error'>: [Errno 104] Connection reset by peer)
HTTPSConnectionPool(host='storage101.ord1.clouddrive.com', port=443): Max retries exceeded with url: /v1/MossoCloudFS_fd6d6695-7fe7-4f77-9b4a-da7696e71dc2/fosdem/veyepar/debian/debconf14/dv/room338/2014-08-25/10_02_05.ogv (Caused by <class 'socket.error'>: [Errno 32] Broken pipe)
"""
print e
# self.ret_text = "rax error: %s" % ( e.body )
import code
code.interact(local=locals())
ret = False
# urllib.quote
# filenames may have chars that need to be quoted for a URL.
# cdn_streaming because.. video? (not sure really)
# self.new_url = container.cdn_streaming_uri +"/"+ urllib.quote(obj.name)
self.new_url = container.cdn_uri +"/"+ urllib.quote(obj.name.encode('utf-8'))
# self.new_url = container.cdn_uri +"/"+ urllib.quote(obj.name)
print "Rackspace: ", self.new_url
return ret
