def put_container(self, container_name): token = self.fetch_token() url = self.auth['endpoint_url'] try: put_container(url=url, token=token, container=container_name) except ClientException: pass
def connect_swift(): http_conn = swiftclient.http_connection(PRE_AUTH_URL) try: swiftclient.head_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn) except swiftclient.ClientException: swiftclient.put_container(PRE_AUTH_URL, PRE_AUTH_TOKEN, CONTAINER_NAME, http_conn=http_conn) return http_conn
def __init__(self): self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=dict( list({ "tenant_id": self.tenant_id, "tenant_name": self.tenant_name }.items()) + list(self.os_extra_options.items())), ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured("Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urllib.parse.urlsplit(self.base_url) split_override = urllib.parse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urllib.parse.urlunsplit(split_result) self.base_url = urllib.parse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def __init__(self, maxfiles, iters, host=benchmachines.swifthost, user='******', key='sagefs', container='bench'): self.host = host self.authurl = self.getAuthUrl() self.user = user self.key = key self.container = container self.sturl, self.token = self.getAuth() ScalabilityTester.__init__(self, maxfiles, iters) swiftclient.put_container(self.sturl, self.token, self.container)
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options={"tenant_name": self.tenant_name}, ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: if self.auto_create_container: swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) else: raise ImproperlyConfigured("Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def __init__(self, logger, conf, names): Bench.__init__(self, logger, conf, names) self.concurrency = self.put_concurrency self.total = self.total_objects self.msg = 'PUTS' if self.use_proxy: with self.connection() as conn: for container_name in self.containers: client.put_container(self.url, self.token, container_name, http_conn=conn)
def __init__(self): self.app = ProxyApplication(None) self.memcache = self._memcache() self.account_name = 'AUTH_%s' % ACCOUNT # prefix AUTH_ plus account name self.token, self.url = get_auth_token() self.container = gen_text(5) resp = {} client.put_container( self.url, self.token, self.container, response_dict=resp) assert resp['status'] == 201
def __init__(self): self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=dict({"tenant_name": self.tenant_name}.items() + self.os_extra_options.items()), cacert=self.custom_ca, insecure=self.ssl_insecure ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) self.last_headers_name = None self.last_headers_value = None # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options={"tenant_name": self.tenant_name}, ) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: if self.auto_create_container: swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2]).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
def create_container_if_not_exists(self,container): """ SWIFT CALL to create container if not exist. If container is created it returns true. """ try: does_container_exist = files.get_container(self.auth.url,self.auth.token,container) log.instance.logger.debug('Container ``' + container + '`` exists already, skipping create... ') return False except files.client.ClientException: createcontainer = files.put_container(self.auth.url,self.auth.token,container) log.instance.logger.info('Container ``' + container + '`` doesnt exist, creating... ') return True
) except (swiftclient.client.ClientException), e: logging.info("ERROR: getting container: %s, %s" % (container_name, e.http_reason)) return try: # Check that the container exists on dest swiftclient.head_container("", dest_token, container_name, http_conn=dest_storage_cnx) except (swiftclient.client.ClientException), e: container_headers = orig_container_headers.copy() for h in ("x-container-object-count", "x-trans-id", "x-container-bytes-used"): del container_headers[h] p = dest_storage_cnx[0] url = "%s://%s%s" % (p.scheme, p.netloc, p.path) try: swiftclient.put_container(url, dest_token, container_name, headers=container_headers) except (swiftclient.client.ClientException), e: logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason)) return try: dest_container_headers, dest_objects = swiftclient.get_container( None, dest_token, container_name, http_conn=dest_storage_cnx ) except (swiftclient.client.ClientException), e: logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason)) return do_headers = False if len(dest_container_headers) != len(orig_container_headers): do_headers = True
container_name, http_conn=dest_storage_cnx) except (swiftclient.client.ClientException), e: container_headers = orig_container_headers.copy() for h in ('x-container-object-count', 'x-trans-id', 'x-container-bytes-used'): try: del container_headers[h] except KeyError: # Nov2013: swift server does not set x-trans-id header pass p = dest_storage_cnx[0] url = "%s://%s%s" % (p.scheme, p.netloc, p.path) try: swiftclient.put_container(url, dest_token, container_name, headers=container_headers) except (swiftclient.client.ClientException), e: logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason)) return try: dest_container_headers, dest_objects = swiftclient.get_container( None, dest_token, container_name, http_conn=dest_storage_cnx, full_listing=True, ) except (swiftclient.client.ClientException), e:
if version_cont: version_cont = quote(version_cont) unset_version = params.get('unset_version_container', None) if action == 'cont_list': (acct_status, cont_list) = get_account(storage_url, token, marker=marker, limit=lines) #resp.app_iter = [json.dumps(c) for c in cont_list] if action == 'cont_create': if cont: if '/' in cont or len(cont) > 254: return HTTP_PRECONDITION_FAILED try: put_container(storage_url, token, cont) except ClientException, err: return err.http_status return HTTP_CREATED return HTTP_BAD_REQUEST if action == 'obj_list': (cont_status, obj_list) = get_container(storage_url, token, cont, marker=marker, limit=lines) #resp.app_iter = [json.dumps(o) for o in obj_list] if action == 'cont_delete': try: delete_container(storage_url, token, cont) except ClientException, err: return err.http_status
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store'] for endpoints in object_store_catalog: if endpoints['region'] == os.getenv('OS_SWIFT_REGION'): region_endpoint = endpoints if not region_endpoint: print "Failing, swift region not found in endpoint list." exit() # Upload our file container_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10)) print "Creating: "+container_name swiftclient.put_container(region_endpoint['publicURL'],token,container_name) agent_file = open("wringer.py") print "Uploading agent: wringer.py" swiftclient.client.put_object(region_endpoint['publicURL'], container=container_name, token=token, name="wringer.py", contents=agent_file.read()) # Request the config url = "swift://%s/%s/%s" % (region_endpoint['region'], container_name, "wringer.py") print "URL: "+url print "Requesting agent config." agent_call = agent_query(token=token,url=api_url, path="/v1.0/tenants/"+tenant_id+"/agent_config/", data=json.dumps({"agent_url":url}), method="POST")
def __init__(self, **settings): # check if some of the settings provided as class attributes # should be overwritten for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) validate_settings(self) self.last_headers_name = None self.last_headers_value = None self.os_options = { 'tenant_id': self.tenant_id, 'tenant_name': self.tenant_name, 'user_domain_id': self.user_domain_id, 'user_domain_name': self.user_domain_name, 'project_domain_id': self.project_domain_id, 'project_domain_name': self.project_domain_name } self.os_options.update(self.os_extra_options) # Get authentication token self.storage_url, self.token = swiftclient.get_auth( self.api_auth_url, self.api_username, self.api_key, auth_version=self.auth_version, os_options=self.os_options) self.http_conn = swiftclient.http_connection(self.storage_url) # Check container try: swiftclient.head_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn) except swiftclient.ClientException: headers = {} if self.auto_create_container: if self.auto_create_container_public: headers['X-Container-Read'] = '.r:*' if self.auto_create_container_allow_orgin: headers['X-Container-Meta-Access-Control-Allow-Origin'] = \ self.auto_create_container_allow_orgin swiftclient.put_container(self.storage_url, self.token, self.container_name, http_conn=self.http_conn, headers=headers) else: raise ImproperlyConfigured( "Container %s does not exist." % self.container_name) if self.auto_base_url: # Derive a base URL based on the authentication information from # the server, optionally overriding the protocol, host/port and # potentially adding a path fragment before the auth information. self.base_url = self.storage_url + '/' if self.override_base_url is not None: # override the protocol and host, append any path fragments split_derived = urlparse.urlsplit(self.base_url) split_override = urlparse.urlsplit(self.override_base_url) split_result = [''] * 5 split_result[0:2] = split_override[0:2] split_result[2] = (split_override[2] + split_derived[2] ).replace('//', '/') self.base_url = urlparse.urlunsplit(split_result) self.base_url = urlparse.urljoin(self.base_url, self.container_name) self.base_url += '/' else: self.base_url = self.override_base_url
for endpoints in object_store_catalog: if endpoints['region'] == os.getenv('OS_SWIFT_REGION'): region_endpoint = endpoints if not region_endpoint: print "Failing, swift region not found in endpoint list." exit() # Upload our file container_name = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(10)) print "Creating: " + container_name swiftclient.put_container(region_endpoint['publicURL'], token, container_name) agent_file = open("wringer.py") print "Uploading agent: wringer.py" swiftclient.client.put_object(region_endpoint['publicURL'], container=container_name, token=token, name="wringer.py", contents=agent_file.read()) # Request the config url = "swift://%s/%s/%s" % (region_endpoint['region'], container_name, "wringer.py") print "URL: " + url
break # Combine with the last tweet storage file contents tweet_list = sorted(tweet_list, key=itemgetter('id')) # Split into monthly files and save tweet_store = {'tweets': {},'mins':{},'maxes':{}} for tweet in tweet_list: time_struct = time.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y") key = time.strftime("tweets-%Y%m.json",time_struct) if not tweet_store['tweets'].get(key): tweet_store['tweets'][key] = [tweet] else: tweet_store['tweets'][key].extend([tweet]) if not tweet_store['mins'].get(key) or tweet_store['mins'].get(key) > tweet['id']: tweet_store['mins'][key] = tweet['id'] if not tweet_store['maxes'].get(key) or tweet_store['maxes'].get(key) < tweet['id']: tweet_store['maxes'][key] = tweet['id'] swiftclient.put_container(region_endpoints['publicURL'],ca.creds['token'],ca.conf['container']) for key in sorted(tweet_store['tweets']): ca.log("Storing file: "+ca.conf['container']+"/"+ca.conf['path']+"/"+key) swiftclient.put_object(region_endpoints['publicURL']+"/"+ca.conf['container']+"/"+ca.conf['path']+"/"+key, token=ca.creds['token'], contents=json.dumps(tweet_store['tweets'][key]), headers={"X-Object-Meta-First-Tweet-ID": tweet_store['mins'][key], "X-Object-Meta-Last-Tweet-Id": tweet_store['maxes'][key]}) ca.run(agent)
# Get our container listing, if the container exists. try: files = get_swift_container(region_endpoints['publicURL']+"/"+ca.conf['container'],ca.creds['token']) except StandardError,e: print "Got error!",e files = [] s3 = S3Connection(ca.conf['s3accesskey'].encode('ascii'), ca.conf['s3secretkey'].encode('ascii')) bucket = s3.get_bucket(ca.conf['s3bucket']) swiftclient.put_container(region_endpoints['publicURL'],ca.creds['token'],ca.conf['container']) max_workers = 5 pool = Pool(max_workers) global copied_files copied_files = [] for key in bucket: #ca.log("Found "+key.name) # if key.name[-1] == "/": # if not key.name[0:-1] in files: # pool.apply_async(worker, args=(region_endpoints['publicURL'], # ca.conf['swiftcontainer'], # ca.creds['token'], # key.name, # ca.conf['s3bucket'],
if version_cont: version_cont = quote(version_cont) unset_version = params.get('unset_version_container', None) if action == 'cont_list': (acct_status, cont_list) = get_account(storage_url, token, marker=marker, limit=lines) #resp.app_iter = [json.dumps(c) for c in cont_list] if action == 'cont_create': if cont: if '/' in cont or len(cont) > 254: return HTTP_PRECONDITION_FAILED try: put_container(storage_url, token, cont) except ClientException, err: return err.http_status return HTTP_CREATED return HTTP_BAD_REQUEST if action == 'obj_list': (cont_status, obj_list) = get_container(storage_url, token, cont, marker=marker, limit=lines) #resp.app_iter = [json.dumps(o) for o in obj_list] if action == 'cont_delete': try: delete_container(storage_url, token, cont) except ClientException, err:
def agent(): ca.log("Starting!") keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'], auth_url=ca.creds['identity_url']) object_store_catalog = keystone.service_catalog.get_endpoints()['object-store'] source_endpoint = None for endpoints in object_store_catalog: if endpoints['region'] == ca.conf['region']: source_endpoint = endpoints if not source_endpoint: ca.log_fail("Failing, source region not found in endpoint list.") exit() target_endpoint = None for endpoints in object_store_catalog: if endpoints['region'] == ca.conf['galleryregion']: target_endpoint = endpoints if not source_endpoint: ca.log_fail("Failing, target region not found in endpoint list.") exit() try: container = swiftclient.head_container(target_endpoint['publicURL'],ca.creds['token'], ca.conf['gallerycontainer']) if not '.r:*' in container.get('x-container-read'): ca.warn("Gallery container exists, but may not be publicly readable.","") except: ca.log("Gallery container doesn't exist, creating new publicly readable container.","") swiftclient.put_container(target_endpoint['publicURL'],ca.creds['token'], ca.conf['gallerycontainer'],{"X-Container-Read": ".r:*"}) ca.log("Getting target listing.","") targetlisting = get_swift_container(target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"?prefix="+ca.conf.get('gallerypath'),ca.creds['token']) ca.log("Getting source listing.","") sourcelisting = get_swift_container(source_endpoint['publicURL']+"/"+ca.conf['container']+"?prefix="+ca.conf.get('path'),ca.creds['token']) good_files = re.compile(".*\.(JPG|jpg|JPEG|jpeg|PNG|png)$") something_changed = False # We should add some stuff to clean up paths, ie: ensure they end in a slash. Later. for file in sourcelisting: if good_files.match(file): justfile = file.replace(ca.conf.get("path"), "") (name,ext) = justfile.rsplit(".",1) if not ca.conf.get("gallerypath")+name+"-large."+ext in targetlisting or not ca.conf.get("gallerypath")+name+"-small."+ext in targetlisting: something_changed = True ca.log("Scaling "+justfile+" to smaller sizes.","") rescale( {"url": source_endpoint['publicURL'],"container": ca.conf['container'],"name":ca.conf.get('path')+justfile}, {"url": target_endpoint['publicURL'],"container": ca.conf['gallerycontainer'],"name":ca.conf.get('gallerypath')+name+"-large."+ext}, {"url": target_endpoint['publicURL'],"container": ca.conf['gallerycontainer'],"name":ca.conf.get('gallerypath')+name+"-small."+ext}, ca.creds['token']) targetlisting.append(ca.conf['gallerypath']+name+"-large."+ext) tree = {} subs = {} if something_changed: # Rebuild our templates. for file in sourcelisting: if good_files.match(file): name = file.replace(ca.conf.get("path"), "") path = name.split("/") if len(path) > 1: subpath = "/".join(path[:-1]) if tree.get(subpath): tree[subpath].extend([path[-1]]) else: tree[subpath] = [path[-1]] else: if tree.get(""): tree[""].extend(path) else: tree[""] = path if len(path) > 1: level_above = "/".join(path[0:-2]) if subs.get(level_above): subs[level_above].add(path[-2]) else: subs[level_above] = set([path[-2]]) head = """ <head><title>%s Gallery</title> <link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/css/bootstrap-combined.min.css" rel="stylesheet"> <link href="https://region-a.geo-1.objects.hpcloudsvc.com:443/v1.0/16026287679679/testcontainer/colorbox/colorbox.css" rel="stylesheet"> <script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.1/jquery.min.js"></script> <script src="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/js/bootstrap.min.js"></script> <script src="http://cdn.jsdelivr.net/colorbox/1.3.19.3/jquery.colorbox-min.js"></script> <script> $(document).ready(function(){ $('a.gallery').colorbox({ rel:'group1' }); }); </script> <style> .thumbnail { height: 150px, width: 150px } </style> </head> <body> <div class="container"> <h1>%s Gallery</h1> """ sublist = """ <h2>Sub-Galleries</h2> <ul class="nav nav-tabs nav-stacked"> """ back_up = """ <ul class="nav nav-tabs nav-stacked"> <li><a href="../">Back to %s</a></li> </ul> """ gallery_top = """ <ul class="thumbnails"> """ gallery_bottom = """ </ul> """ footer = """ </div> </body> """ for name, template in tree.items(): ca.log("Writing gallery template for "+name,"") pretty_name = name.split("/")[-1] html = head % (pretty_name,pretty_name) if name: if len(name.split("/")) == 1: html += back_up % ("Home") elif len(name.split("/")) > 1: html += back_up % (name.split("/")[-2]) if subs.get(name): html += sublist for subpath in subs[name]: html += '<li><a href="%s/">%s</a></li>\n' % (subpath, subpath) html += "</ul>\n" html += gallery_top for file in template: (filename,extension) = file.rsplit(".",1) html += "<li class='span3'>\n<a href='%s' class='gallery thumbnail'><img src='%s'></a></li>\n" % (filename+"-large."+extension,filename+"-small."+extension) html += gallery_bottom html += footer filename = name if filename and filename[-1] != '/': filename = filename +"/" swiftclient.put_object(target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"/"+ca.conf.get('gallerypath')+filename, contents=html, content_type="text/html", token=ca.creds['token']) ca.log("Gallery updated at "+target_endpoint['publicURL']+"/"+ca.conf['gallerycontainer']+"/"+ca.conf.get('gallerypath'),"") else: ca.log("No new files found.","")
def agent(): ca.log("Starting!") keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'], auth_url=ca.creds['identity_url']) object_store_catalog = keystone.service_catalog.get_endpoints( )['object-store'] source_endpoint = None for endpoints in object_store_catalog: if endpoints['region'] == ca.conf['region']: source_endpoint = endpoints if not source_endpoint: ca.log_fail("Failing, source region not found in endpoint list.") exit() target_endpoint = None for endpoints in object_store_catalog: if endpoints['region'] == ca.conf['galleryregion']: target_endpoint = endpoints if not source_endpoint: ca.log_fail("Failing, target region not found in endpoint list.") exit() try: container = swiftclient.head_container(target_endpoint['publicURL'], ca.creds['token'], ca.conf['gallerycontainer']) if not '.r:*' in container.get('x-container-read'): ca.warn( "Gallery container exists, but may not be publicly readable.", "") except: ca.log( "Gallery container doesn't exist, creating new publicly readable container.", "") swiftclient.put_container(target_endpoint['publicURL'], ca.creds['token'], ca.conf['gallerycontainer'], {"X-Container-Read": ".r:*"}) ca.log("Getting target listing.", "") targetlisting = get_swift_container( target_endpoint['publicURL'] + "/" + ca.conf['gallerycontainer'] + "?prefix=" + ca.conf.get('gallerypath'), ca.creds['token']) ca.log("Getting source listing.", "") sourcelisting = get_swift_container( source_endpoint['publicURL'] + "/" + ca.conf['container'] + "?prefix=" + ca.conf.get('path'), ca.creds['token']) good_files = re.compile(".*\.(JPG|jpg|JPEG|jpeg|PNG|png)$") something_changed = False # We should add some stuff to clean up paths, ie: ensure they end in a slash. Later. for file in sourcelisting: if good_files.match(file): justfile = file.replace(ca.conf.get("path"), "") (name, ext) = justfile.rsplit(".", 1) if not ca.conf.get( "gallerypath" ) + name + "-large." + ext in targetlisting or not ca.conf.get( "gallerypath") + name + "-small." + ext in targetlisting: something_changed = True ca.log("Scaling " + justfile + " to smaller sizes.", "") rescale( { "url": source_endpoint['publicURL'], "container": ca.conf['container'], "name": ca.conf.get('path') + justfile }, { "url": target_endpoint['publicURL'], "container": ca.conf['gallerycontainer'], "name": ca.conf.get('gallerypath') + name + "-large." + ext }, { "url": target_endpoint['publicURL'], "container": ca.conf['gallerycontainer'], "name": ca.conf.get('gallerypath') + name + "-small." + ext }, ca.creds['token']) targetlisting.append(ca.conf['gallerypath'] + name + "-large." + ext) tree = {} subs = {} if something_changed: # Rebuild our templates. for file in sourcelisting: if good_files.match(file): name = file.replace(ca.conf.get("path"), "") path = name.split("/") if len(path) > 1: subpath = "/".join(path[:-1]) if tree.get(subpath): tree[subpath].extend([path[-1]]) else: tree[subpath] = [path[-1]] else: if tree.get(""): tree[""].extend(path) else: tree[""] = path if len(path) > 1: level_above = "/".join(path[0:-2]) if subs.get(level_above): subs[level_above].add(path[-2]) else: subs[level_above] = set([path[-2]]) head = """ <head><title>%s Gallery</title> <link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/css/bootstrap-combined.min.css" rel="stylesheet"> <link href="https://region-a.geo-1.objects.hpcloudsvc.com:443/v1.0/16026287679679/testcontainer/colorbox/colorbox.css" rel="stylesheet"> <script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.1/jquery.min.js"></script> <script src="//netdna.bootstrapcdn.com/twitter-bootstrap/2.1.1/js/bootstrap.min.js"></script> <script src="http://cdn.jsdelivr.net/colorbox/1.3.19.3/jquery.colorbox-min.js"></script> <script> $(document).ready(function(){ $('a.gallery').colorbox({ rel:'group1' }); }); </script> <style> .thumbnail { height: 150px, width: 150px } </style> </head> <body> <div class="container"> <h1>%s Gallery</h1> """ sublist = """ <h2>Sub-Galleries</h2> <ul class="nav nav-tabs nav-stacked"> """ back_up = """ <ul class="nav nav-tabs nav-stacked"> <li><a href="../">Back to %s</a></li> </ul> """ gallery_top = """ <ul class="thumbnails"> """ gallery_bottom = """ </ul> """ footer = """ </div> </body> """ for name, template in tree.items(): ca.log("Writing gallery template for " + name, "") pretty_name = name.split("/")[-1] html = head % (pretty_name, pretty_name) if name: if len(name.split("/")) == 1: html += back_up % ("Home") elif len(name.split("/")) > 1: html += back_up % (name.split("/")[-2]) if subs.get(name): html += sublist for subpath in subs[name]: html += '<li><a href="%s/">%s</a></li>\n' % (subpath, subpath) html += "</ul>\n" html += gallery_top for file in template: (filename, extension) = file.rsplit(".", 1) html += "<li class='span3'>\n<a href='%s' class='gallery thumbnail'><img src='%s'></a></li>\n" % ( filename + "-large." + extension, filename + "-small." + extension) html += gallery_bottom html += footer filename = name if filename and filename[-1] != '/': filename = filename + "/" swiftclient.put_object(target_endpoint['publicURL'] + "/" + ca.conf['gallerycontainer'] + "/" + ca.conf.get('gallerypath') + filename, contents=html, content_type="text/html", token=ca.creds['token']) ca.log( "Gallery updated at " + target_endpoint['publicURL'] + "/" + ca.conf['gallerycontainer'] + "/" + ca.conf.get('gallerypath'), "") else: ca.log("No new files found.", "")