def put_container(self, container_name):
token = self.fetch_token()
url = self.auth['endpoint_url']
try:
put_container(url=url, token=token, container=container_name)
except ClientException:
pass
def connect_swift():
http_conn = swiftclient.http_connection(PRE_AUTH_URL)
try:
swiftclient.head_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn)
except swiftclient.ClientException:
swiftclient.put_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn)
return http_conn
def __init__(self):
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=dict(
list({
"tenant_id": self.tenant_id,
"tenant_name": self.tenant_name
}.items()) + list(self.os_extra_options.items())),
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured("Container %s does not exist." %
self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urllib.parse.urlsplit(self.base_url)
split_override = urllib.parse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urllib.parse.urlunsplit(split_result)
self.base_url = urllib.parse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def __init__(self, maxfiles, iters, host=benchmachines.swifthost, user='******', key='sagefs', container='bench'): self.host = host self.authurl = self.getAuthUrl() self.user = user self.key = key self.container = container self.sturl, self.token = self.getAuth() ScalabilityTester.__init__(self, maxfiles, iters) swiftclient.put_container(self.sturl, self.token, self.container)
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options={"tenant_name": self.tenant_name},
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
if self.auto_create_container:
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
else:
raise ImproperlyConfigured("Container %s does not exist." %
self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def __init__(self, logger, conf, names):
Bench.__init__(self, logger, conf, names)
self.concurrency = self.put_concurrency
self.total = self.total_objects
self.msg = 'PUTS'
if self.use_proxy:
with self.connection() as conn:
for container_name in self.containers:
client.put_container(self.url, self.token,
container_name, http_conn=conn)
def __init__(self):
self.app = ProxyApplication(None)
self.memcache = self._memcache()
self.account_name = 'AUTH_%s' % ACCOUNT # prefix AUTH_ plus account name
self.token, self.url = get_auth_token()
self.container = gen_text(5)
resp = {}
client.put_container(
self.url, self.token, self.container, response_dict=resp)
assert resp['status'] == 201
def __init__(self):
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=dict({"tenant_name": self.tenant_name}.items() +
self.os_extra_options.items()),
cacert=self.custom_ca,
insecure=self.ssl_insecure
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
swiftclient.put_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options={"tenant_name": self.tenant_name},
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
if self.auto_create_container:
swiftclient.put_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def __init__(self, logger, conf, names):
Bench.__init__(self, logger, conf, names)
self.concurrency = self.put_concurrency
self.total = self.total_objects
self.msg = 'PUTS'
if self.use_proxy:
with self.connection() as conn:
for container_name in self.containers:
client.put_container(self.url,
self.token,
container_name,
http_conn=conn)
def create_container_if_not_exists(self,container):
""" SWIFT CALL to create container if not exist. If container is created it returns true. """
try:
does_container_exist = files.get_container(self.auth.url,self.auth.token,container)
log.instance.logger.debug('Container ``' + container + '`` exists already, skipping create... ')
return False
except files.client.ClientException:
createcontainer = files.put_container(self.auth.url,self.auth.token,container)
log.instance.logger.info('Container ``' + container + '`` doesnt exist, creating... ')
return True
)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: getting container: %s, %s" % (container_name, e.http_reason))
return
try:
# Check that the container exists on dest
swiftclient.head_container("", dest_token, container_name, http_conn=dest_storage_cnx)
except (swiftclient.client.ClientException), e:
container_headers = orig_container_headers.copy()
for h in ("x-container-object-count", "x-trans-id", "x-container-bytes-used"):
del container_headers[h]
p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try:
swiftclient.put_container(url, dest_token, container_name, headers=container_headers)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason))
return
try:
dest_container_headers, dest_objects = swiftclient.get_container(
None, dest_token, container_name, http_conn=dest_storage_cnx
)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason))
return
do_headers = False
if len(dest_container_headers) != len(orig_container_headers):
do_headers = True
container_name,
http_conn=dest_storage_cnx)
except (swiftclient.client.ClientException), e:
container_headers = orig_container_headers.copy()
for h in ('x-container-object-count', 'x-trans-id',
'x-container-bytes-used'):
try:
del container_headers[h]
except KeyError:
# Nov2013: swift server does not set x-trans-id header
pass
p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try:
swiftclient.put_container(url,
dest_token,
container_name,
headers=container_headers)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" %
(container_name, e.http_reason))
return
try:
dest_container_headers, dest_objects = swiftclient.get_container(
None,
dest_token,
container_name,
http_conn=dest_storage_cnx,
full_listing=True,
)
except (swiftclient.client.ClientException), e:
if version_cont:
version_cont = quote(version_cont)
unset_version = params.get('unset_version_container', None)
if action == 'cont_list':
(acct_status, cont_list) = get_account(storage_url,
token,
marker=marker,
limit=lines)
#resp.app_iter = [json.dumps(c) for c in cont_list]
if action == 'cont_create':
if cont:
if '/' in cont or len(cont) > 254:
return HTTP_PRECONDITION_FAILED
try:
put_container(storage_url, token, cont)
except ClientException, err:
return err.http_status
return HTTP_CREATED
return HTTP_BAD_REQUEST
if action == 'obj_list':
(cont_status, obj_list) = get_container(storage_url,
token, cont,
marker=marker,
limit=lines)
#resp.app_iter = [json.dumps(o) for o in obj_list]
if action == 'cont_delete':
try:
delete_container(storage_url, token, cont)
except ClientException, err:
return err.http_status
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
for endpoints in object_store_catalog:
if endpoints['region'] == os.getenv('OS_SWIFT_REGION'):
region_endpoint = endpoints
if not region_endpoint:
print "Failing, swift region not found in endpoint list."
exit()
# Upload our file
container_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
print "Creating: "+container_name
swiftclient.put_container(region_endpoint['publicURL'],token,container_name)
agent_file = open("wringer.py")
print "Uploading agent: wringer.py"
swiftclient.client.put_object(region_endpoint['publicURL'], container=container_name,
token=token, name="wringer.py", contents=agent_file.read())
# Request the config
url = "swift://%s/%s/%s" % (region_endpoint['region'], container_name, "wringer.py")
print "URL: "+url
print "Requesting agent config."
agent_call = agent_query(token=token,url=api_url,
path="/v1.0/tenants/"+tenant_id+"/agent_config/",
data=json.dumps({"agent_url":url}), method="POST")
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
validate_settings(self)
self.last_headers_name = None
self.last_headers_value = None
self.os_options = {
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'user_domain_id': self.user_domain_id,
'user_domain_name': self.user_domain_name,
'project_domain_id': self.project_domain_id,
'project_domain_name': self.project_domain_name
}
self.os_options.update(self.os_extra_options)
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=self.os_options)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
if self.auto_create_container_allow_orgin:
headers['X-Container-Meta-Access-Control-Allow-Origin'] = \
self.auto_create_container_allow_orgin
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] + split_derived[2]
).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
for endpoints in object_store_catalog:
if endpoints['region'] == os.getenv('OS_SWIFT_REGION'):
region_endpoint = endpoints
if not region_endpoint:
print "Failing, swift region not found in endpoint list."
exit()
# Upload our file
container_name = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(10))
print "Creating: " + container_name
swiftclient.put_container(region_endpoint['publicURL'], token,
container_name)
agent_file = open("wringer.py")
print "Uploading agent: wringer.py"
swiftclient.client.put_object(region_endpoint['publicURL'],
container=container_name,
token=token,
name="wringer.py",
contents=agent_file.read())
# Request the config
url = "swift://%s/%s/%s" % (region_endpoint['region'], container_name,
"wringer.py")
print "URL: " + url
break
# Combine with the last tweet storage file contents
tweet_list = sorted(tweet_list, key=itemgetter('id'))
# Split into monthly files and save
tweet_store = {'tweets': {},'mins':{},'maxes':{}}
for tweet in tweet_list:
time_struct = time.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")
key = time.strftime("tweets-%Y%m.json",time_struct)
if not tweet_store['tweets'].get(key):
tweet_store['tweets'][key] = [tweet]
else:
tweet_store['tweets'][key].extend([tweet])
if not tweet_store['mins'].get(key) or tweet_store['mins'].get(key) > tweet['id']:
tweet_store['mins'][key] = tweet['id']
if not tweet_store['maxes'].get(key) or tweet_store['maxes'].get(key) < tweet['id']:
tweet_store['maxes'][key] = tweet['id']
swiftclient.put_container(region_endpoints['publicURL'],ca.creds['token'],ca.conf['container'])
for key in sorted(tweet_store['tweets']):
ca.log("Storing file: "+ca.conf['container']+"/"+ca.conf['path']+"/"+key)
swiftclient.put_object(region_endpoints['publicURL']+"/"+ca.conf['container']+"/"+ca.conf['path']+"/"+key, token=ca.creds['token'], contents=json.dumps(tweet_store['tweets'][key]), headers={"X-Object-Meta-First-Tweet-ID": tweet_store['mins'][key], "X-Object-Meta-Last-Tweet-Id": tweet_store['maxes'][key]})
ca.run(agent)
# Get our container listing, if the container exists.
try:
files = get_swift_container(region_endpoints['publicURL']+"/"+ca.conf['container'],ca.creds['token'])
except StandardError,e:
print "Got error!",e
files = []
s3 = S3Connection(ca.conf['s3accesskey'].encode('ascii'), ca.conf['s3secretkey'].encode('ascii'))
bucket = s3.get_bucket(ca.conf['s3bucket'])
swiftclient.put_container(region_endpoints['publicURL'],ca.creds['token'],ca.conf['container'])
max_workers = 5
pool = Pool(max_workers)
global copied_files
copied_files = []
for key in bucket:
#ca.log("Found "+key.name)
# if key.name[-1] == "/":
# if not key.name[0:-1] in files:
# pool.apply_async(worker, args=(region_endpoints['publicURL'],
# ca.conf['swiftcontainer'],
# ca.creds['token'],
# key.name,
# ca.conf['s3bucket'],
if version_cont:
version_cont = quote(version_cont)
unset_version = params.get('unset_version_container', None)
if action == 'cont_list':
(acct_status, cont_list) = get_account(storage_url,
token,
marker=marker,
limit=lines)
#resp.app_iter = [json.dumps(c) for c in cont_list]
if action == 'cont_create':
if cont:
if '/' in cont or len(cont) > 254:
return HTTP_PRECONDITION_FAILED
try:
put_container(storage_url, token, cont)
except ClientException, err:
return err.http_status
return HTTP_CREATED
return HTTP_BAD_REQUEST
if action == 'obj_list':
(cont_status, obj_list) = get_container(storage_url,
token,
cont,
marker=marker,
limit=lines)
#resp.app_iter = [json.dumps(o) for o in obj_list]
if action == 'cont_delete':
try:
delete_container(storage_url, token, cont)
except ClientException, err:
def agent():
ca.log("Starting!")
keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
source_endpoint = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
source_endpoint = endpoints
if not source_endpoint:
ca.log_fail("Failing, source region not found in endpoint list.")
exit()
target_endpoint = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['galleryregion']:
target_endpoint = endpoints
if not source_endpoint:
ca.log_fail("Failing, target region not found in endpoint list.")
exit()
try:
container = swiftclient.head_container(target_endpoint['publicURL'],ca.creds['token'],
ca.conf['gallerycontainer'])
if not '.r:*' in container.get('x-container-read'):
ca.warn("Gallery container exists, but may not be publicly readable.","")
except:
ca.log("Gallery container doesn't exist, creating new publicly readable container.","")
swiftclient.put_container(target_endpoint['publicURL'],ca.creds['token'],
ca.conf['gallerycontainer'],{"X-Container-Read": ".r:*"})
ca.log("Getting target listing.","")
targetlisting = get_swift_container(target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"?prefix="+ca.conf.get('gallerypath'),ca.creds['token'])
ca.log("Getting source listing.","")
sourcelisting = get_swift_container(source_endpoint['publicURL']+"/"+ca.conf['container']+"?prefix="+ca.conf.get('path'),ca.creds['token'])
good_files = re.compile(".*\.(JPG|jpg|JPEG|jpeg|PNG|png)$")
something_changed = False
# We should add some stuff to clean up paths, ie: ensure they end in a slash. Later.
for file in sourcelisting:
if good_files.match(file):
justfile = file.replace(ca.conf.get("path"), "")
(name,ext) = justfile.rsplit(".",1)
if not ca.conf.get("gallerypath")+name+"-large."+ext in targetlisting or not ca.conf.get("gallerypath")+name+"-small."+ext in targetlisting:
something_changed = True
ca.log("Scaling "+justfile+" to smaller sizes.","")
rescale(
{"url": source_endpoint['publicURL'],"container": ca.conf['container'],"name":ca.conf.get('path')+justfile},
{"url": target_endpoint['publicURL'],"container": ca.conf['gallerycontainer'],"name":ca.conf.get('gallerypath')+name+"-large."+ext},
{"url": target_endpoint['publicURL'],"container": ca.conf['gallerycontainer'],"name":ca.conf.get('gallerypath')+name+"-small."+ext},
ca.creds['token'])
targetlisting.append(ca.conf['gallerypath']+name+"-large."+ext)
tree = {}
subs = {}
if something_changed:
# Rebuild our templates.
for file in sourcelisting:
if good_files.match(file):
name = file.replace(ca.conf.get("path"), "")
path = name.split("/")
if len(path) > 1:
subpath = "/".join(path[:-1])
if tree.get(subpath):
tree[subpath].extend([path[-1]])
else:
tree[subpath] = [path[-1]]
else:
if tree.get(""):
tree[""].extend(path)
else:
tree[""] = path
if len(path) > 1:
level_above = "/".join(path[0:-2])
if subs.get(level_above):
subs[level_above].add(path[-2])
else:
subs[level_above] = set([path[-2]])
head = """
<head><title>%s Gallery</title>
<link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/css/bootstrap-combined.min.css" rel="stylesheet">
<link href="https://region-a.geo-1.objects.hpcloudsvc.com:443/v1.0/16026287679679/testcontainer/colorbox/colorbox.css" rel="stylesheet">
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.1/jquery.min.js"></script>
<script src="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/js/bootstrap.min.js"></script>
<script src="http://cdn.jsdelivr.net/colorbox/1.3.19.3/jquery.colorbox-min.js"></script>
<script>
$(document).ready(function(){
$('a.gallery').colorbox({ rel:'group1' });
});
</script>
<style>
.thumbnail { height: 150px, width: 150px }
</style>
</head>
<body>
<div class="container">
<h1>%s Gallery</h1>
"""
sublist = """
<h2>Sub-Galleries</h2>
<ul class="nav nav-tabs nav-stacked">
"""
back_up = """
<ul class="nav nav-tabs nav-stacked">
<li><a href="../">Back to %s</a></li>
</ul>
"""
gallery_top = """
<ul class="thumbnails">
"""
gallery_bottom = """
</ul>
"""
footer = """
</div>
</body>
"""
for name, template in tree.items():
ca.log("Writing gallery template for "+name,"")
pretty_name = name.split("/")[-1]
html = head % (pretty_name,pretty_name)
if name:
if len(name.split("/")) == 1:
html += back_up % ("Home")
elif len(name.split("/")) > 1:
html += back_up % (name.split("/")[-2])
if subs.get(name):
html += sublist
for subpath in subs[name]:
html += '<li><a href="%s/">%s</a></li>\n' % (subpath, subpath)
html += "</ul>\n"
html += gallery_top
for file in template:
(filename,extension) = file.rsplit(".",1)
html += "<li class='span3'>\n<a href='%s' class='gallery thumbnail'><img src='%s'></a></li>\n" % (filename+"-large."+extension,filename+"-small."+extension)
html += gallery_bottom
html += footer
filename = name
if filename and filename[-1] != '/':
filename = filename +"/"
swiftclient.put_object(target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"/"+ca.conf.get('gallerypath')+filename,
contents=html, content_type="text/html", token=ca.creds['token'])
ca.log("Gallery updated at "+target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"/"+ca.conf.get('gallerypath'),"")
else:
ca.log("No new files found.","")
def agent():
ca.log("Starting!")
keystone = client.Client(token=ca.creds['token'],
tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints(
)['object-store']
source_endpoint = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
source_endpoint = endpoints
if not source_endpoint:
ca.log_fail("Failing, source region not found in endpoint list.")
exit()
target_endpoint = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['galleryregion']:
target_endpoint = endpoints
if not source_endpoint:
ca.log_fail("Failing, target region not found in endpoint list.")
exit()
try:
container = swiftclient.head_container(target_endpoint['publicURL'],
ca.creds['token'],
ca.conf['gallerycontainer'])
if not '.r:*' in container.get('x-container-read'):
ca.warn(
"Gallery container exists, but may not be publicly readable.",
"")
except:
ca.log(
"Gallery container doesn't exist, creating new publicly readable container.",
"")
swiftclient.put_container(target_endpoint['publicURL'],
ca.creds['token'],
ca.conf['gallerycontainer'],
{"X-Container-Read": ".r:*"})
ca.log("Getting target listing.", "")
targetlisting = get_swift_container(
target_endpoint['publicURL'] + "/" + ca.conf['gallerycontainer'] +
"?prefix=" + ca.conf.get('gallerypath'), ca.creds['token'])
ca.log("Getting source listing.", "")
sourcelisting = get_swift_container(
source_endpoint['publicURL'] + "/" + ca.conf['container'] +
"?prefix=" + ca.conf.get('path'), ca.creds['token'])
good_files = re.compile(".*\.(JPG|jpg|JPEG|jpeg|PNG|png)$")
something_changed = False
# We should add some stuff to clean up paths, ie: ensure they end in a slash. Later.
for file in sourcelisting:
if good_files.match(file):
justfile = file.replace(ca.conf.get("path"), "")
(name, ext) = justfile.rsplit(".", 1)
if not ca.conf.get(
"gallerypath"
) + name + "-large." + ext in targetlisting or not ca.conf.get(
"gallerypath") + name + "-small." + ext in targetlisting:
something_changed = True
ca.log("Scaling " + justfile + " to smaller sizes.", "")
rescale(
{
"url": source_endpoint['publicURL'],
"container": ca.conf['container'],
"name": ca.conf.get('path') + justfile
}, {
"url": target_endpoint['publicURL'],
"container": ca.conf['gallerycontainer'],
"name":
ca.conf.get('gallerypath') + name + "-large." + ext
}, {
"url": target_endpoint['publicURL'],
"container": ca.conf['gallerycontainer'],
"name":
ca.conf.get('gallerypath') + name + "-small." + ext
}, ca.creds['token'])
targetlisting.append(ca.conf['gallerypath'] + name +
"-large." + ext)
tree = {}
subs = {}
if something_changed:
# Rebuild our templates.
for file in sourcelisting:
if good_files.match(file):
name = file.replace(ca.conf.get("path"), "")
path = name.split("/")
if len(path) > 1:
subpath = "/".join(path[:-1])
if tree.get(subpath):
tree[subpath].extend([path[-1]])
else:
tree[subpath] = [path[-1]]
else:
if tree.get(""):
tree[""].extend(path)
else:
tree[""] = path
if len(path) > 1:
level_above = "/".join(path[0:-2])
if subs.get(level_above):
subs[level_above].add(path[-2])
else:
subs[level_above] = set([path[-2]])
head = """
<head><title>%s Gallery</title>
<link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/css/bootstrap-combined.min.css" rel="stylesheet">
<link href="https://region-a.geo-1.objects.hpcloudsvc.com:443/v1.0/16026287679679/testcontainer/colorbox/colorbox.css" rel="stylesheet">
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.1/jquery.min.js"></script>
<script src="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/js/bootstrap.min.js"></script>
<script src="http://cdn.jsdelivr.net/colorbox/1.3.19.3/jquery.colorbox-min.js"></script>
<script>
$(document).ready(function(){
$('a.gallery').colorbox({ rel:'group1' });
});
</script>
<style>
.thumbnail { height: 150px, width: 150px }
</style>
</head>
<body>
<div class="container">
<h1>%s Gallery</h1>
"""
sublist = """
<h2>Sub-Galleries</h2>
<ul class="nav nav-tabs nav-stacked">
"""
back_up = """
<ul class="nav nav-tabs nav-stacked">
<li><a href="../">Back to %s</a></li>
</ul>
"""
gallery_top = """
<ul class="thumbnails">
"""
gallery_bottom = """
</ul>
"""
footer = """
</div>
</body>
"""
for name, template in tree.items():
ca.log("Writing gallery template for " + name, "")
pretty_name = name.split("/")[-1]
html = head % (pretty_name, pretty_name)
if name:
if len(name.split("/")) == 1:
html += back_up % ("Home")
elif len(name.split("/")) > 1:
html += back_up % (name.split("/")[-2])
if subs.get(name):
html += sublist
for subpath in subs[name]:
html += '<li><a href="%s/">%s</a></li>\n' % (subpath,
subpath)
html += "</ul>\n"
html += gallery_top
for file in template:
(filename, extension) = file.rsplit(".", 1)
html += "<li class='span3'>\n<a href='%s' class='gallery thumbnail'><img src='%s'></a></li>\n" % (
filename + "-large." + extension,
filename + "-small." + extension)
html += gallery_bottom
html += footer
filename = name
if filename and filename[-1] != '/':
filename = filename + "/"
swiftclient.put_object(target_endpoint['publicURL'] + "/" +
ca.conf['gallerycontainer'] + "/" +
ca.conf.get('gallerypath') + filename,
contents=html,
content_type="text/html",
token=ca.creds['token'])
ca.log(
"Gallery updated at " + target_endpoint['publicURL'] + "/" +
ca.conf['gallerycontainer'] + "/" + ca.conf.get('gallerypath'), "")
else:
ca.log("No new files found.", "")
