def delete_container(self, dest_storage_cnx, dest_token, orig_containers,
dest_containers):
set1 = set((x['name']) for x in orig_containers)
set2 = set((x['name']) for x in dest_containers)
delete_diff = set2 - set1
pool = eventlet.GreenPool(size=self.concurrency)
pile = eventlet.GreenPile(pool)
for container in delete_diff:
try:
dest_container_stats, dest_objects = swiftclient.get_container(
None,
dest_token,
container,
http_conn=dest_storage_cnx,
full_listing=True,
)
except (swiftclient.client.ClientException), e:
logging.info("error getting container: %s, %s" %
(container, e.http_reason))
continue
for obj in dest_objects:
logging.info("deleting obj: %s ts:%s", obj['name'],
obj['last_modified'])
pile.spawn(self.delete_object, dest_storage_cnx, dest_token,
container, obj['name'])
pool.waitall()
logging.info("deleting container: %s", container)
pile.spawn(swiftclient.delete_container,
'',
dest_token,
container,
http_conn=dest_storage_cnx)
def delete_container(self, dest_storage_cnx, dest_token,
orig_containers,
dest_containers):
set1 = set((x['name']) for x in orig_containers)
set2 = set((x['name']) for x in dest_containers)
delete_diff = set2 - set1
pool = eventlet.GreenPool(size=self.concurrency)
pile = eventlet.GreenPile(pool)
for container in delete_diff:
try:
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container, http_conn=dest_storage_cnx,
full_listing=True,
)
except(swiftclient.client.ClientException), e:
logging.info("error getting container: %s, %s" % (
container, e.http_reason))
continue
for obj in dest_objects:
logging.info("deleting obj: %s ts:%s", obj['name'],
obj['last_modified'])
pile.spawn(self.delete_object,
dest_storage_cnx,
dest_token,
container,
obj['name'])
pool.waitall()
logging.info("deleting container: %s", container)
pile.spawn(swiftclient.delete_container,
'', dest_token, container, http_conn=dest_storage_cnx)
def index_container(self, account, container, marker=None, verify=False):
"List all objects for the container and create task for each"
self.logger.debug("Indexing container %s/%s; marker=%s" %
(account.encode('utf8'), container.encode('utf8'),
(marker or '').encode('utf8')))
# Index the container up to marker
url = '/'.join(['http://127.0.0.1', 'v1', account])
headers, objects = get_container(url,
self.auth_token,
container,
limit=self.container_listing_count,
marker=marker,
http_conn=self._get_conn(url))
self.logger.info("Found %s Objects(s) in container %s" %
(len(objects), container.encode('utf8')))
if len(objects) == self.container_listing_count:
self.indexer.publish_backfill_task('container',
account,
container,
marker=objects[-1]['name'],
verify=verify)
update_handler = ContainerUpdateHandler(self.indexer,
account,
container,
index=self.index)
_id, props = update_handler.parse_props(headers)
if verify:
# Get Indexed State of Container
client = get_search_client(self.index_config)
results = client.search(query=pyes.TermQuery("_id", _id),
indices=[
self.index_config.get(
'search_index_name', 'os_default')
])
assert results.total == 1, '%s indexed instead of 1' % \
results.total
for name, val in results[0].iteritems():
if encode_utf8(val) != encode_utf8(props[name]):
self.logger.error('Indexed Property Does Not Match')
self.logger.error(val)
self.logger.error(props[name])
else:
update_handler.index(_id, props, block=True)
# List all of the objects in the container
self.logger.info(
"Found %s Objects in %s/%s" %
(len(objects), account.encode('utf8'), container.encode('utf8')))
for obj in objects:
self.indexer.publish_backfill_task('object',
account,
container,
obj['name'],
verify=verify)
def agent():
keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
object_store_catalog = keystone.service_catalog.get_endpoints()['object-store']
region_endpoints = None
for endpoints in object_store_catalog:
if endpoints['region'] == ca.conf['region']:
region_endpoints = endpoints
if not region_endpoints:
ca.log_fail("Failing, region not found in endpoint list.")
exit()
t = Twython()
# Figure out what files already exist, and what our latest tweet is.
files = []
try:
(headers,files) = swiftclient.get_container(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],full_listing=True, prefix=ca.conf['path'])
except swiftclient.client.ClientException:
pass
files = sorted(files, key=itemgetter('name'))
last_tweet = 0
last_file = ''
tweet_list = []
if files:
(headers,last_file) = swiftclient.get_object(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],files[-1]['name'])
headers = swiftclient.head_object(region_endpoints['publicURL'],ca.creds['token'],
ca.conf['container'],files[-1]['name'])
last_tweet = headers.get('x-object-meta-last-tweet-id',0)
tweet_list = json.loads(last_file)
# Grab our tweet list (tweets since last tweet up to 3200), optimized for
# fewest requests.
try:
if last_tweet:
tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, since_id=last_tweet, include_rts=True)
else:
tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, include_rts=True)
except TwythonError, e:
ca.log_fail("Error accessing twitter stream. User not found or twitter down.")
exit()
def rmtree(self, abs_path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
for obj in container[1]:
if obj['name'].startswith(abs_path):
swiftclient.delete_object(self.storage_url,
token=self.token,
container=self.container_name,
name=obj['name'])
def rmtree(self, abs_path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
for obj in container[1]:
if obj['name'].startswith(abs_path):
swiftclient.delete_object(self.storage_url,
token=self.token,
container=self.container_name,
name=obj['name'])
def create_container_if_not_exists(self,container):
""" SWIFT CALL to create container if not exist. If container is created it returns true. """
try:
does_container_exist = files.get_container(self.auth.url,self.auth.token,container)
log.instance.logger.debug('Container ``' + container + '`` exists already, skipping create... ')
return False
except files.client.ClientException:
createcontainer = files.put_container(self.auth.url,self.auth.token,container)
log.instance.logger.info('Container ``' + container + '`` doesnt exist, creating... ')
return True
def container(container_name):
if 'url' in session and 'token' in session:
session['container'] = container_name
try:
output = swiftclient.get_container(url=session['url'], token=session['token'], container=container_name)
except swiftclient.ClientException:
return redirect(url_for('logout'))
print 'DEBUG: views.container().swiftclient.get_container: '+str(output)
container_info = output[0]
container_info['x-container-bytes-used'] = float(container_info['x-container-bytes-used']) /(1024**3)
for i in output[1]:
i['last_modified'] = i['last_modified'][:10]+" "+i['last_modified'][11:16]
i['content_type'] = i['content_type'].split('/')[1]
temp_meta = Objects.read_temp_url(url=session['url'], token=session['token'], container=container_name, object=i['name'])
i['temp_url'] = temp_meta[0]
timestamp = temp_meta[1]
if timestamp == False:
i['temp_url_exp'] = '- '
else:
i['temp_url_exp'] = datetime.datetime.fromtimestamp(float(timestamp))
print '###############################################################'
obj_list = output[1]
print type(output)
print output[0]
print output[1]
cont_acl = Containers.check_read_acl(url=session['url'], token=session['token'], container_name=container_name)
return render_template('container.html',
title='Container',
container_info=container_info,
obj_list=obj_list,
cont_acl=cont_acl)
'''
result = {'status': output}
return jsonify(**result)
cont_list = Containers().list(session['url'], session['token'])
objs_list = Objects().list(session['url'], session['token'], container_name)
if not objs_list:
return render_template('account.html',
title='Account',
cont_list=cont_list)
return render_template('account.html',
title='container',
objs_list=objs_list,
cont_list=cont_list)
'''
else:
return redirect(url_for('login'))
def sync(self, orig_storage_cnx, orig_storage_url,
orig_token, dest_storage_cnx, dest_storage_url, dest_token,
container_name):
try:
orig_container_headers, orig_objects = swiftclient.get_container(
None, orig_token, container_name, http_conn=orig_storage_cnx,
full_listing=True,
)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: getting container: %s, %s" % (
container_name, e.http_reason))
return
def sync(self, orig_storage_cnx, orig_storage_url, orig_token,
dest_storage_cnx, dest_storage_url, dest_token, container_name):
try:
orig_container_headers, orig_objects = swiftclient.get_container(
None,
orig_token,
container_name,
http_conn=orig_storage_cnx,
)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: getting container: %s, %s" %
(container_name, e.http_reason))
return
def listdir(self, path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name, prefix=path,
full_listing=self.full_listing)
files = []
dirs = []
for obj in container[1]:
remaining_path = obj['name'][len(path):].split('/')
key = remaining_path[0] if remaining_path[0] else remaining_path[1]
if not self.isdir(key):
files.append(key)
elif key not in dirs:
dirs.append(key)
return dirs, files
def ls(self):
""" SWIFT COMMON CLIENT factored in """
full_filelist = [] # List which we'll page through until we get all filenames. We store all file metadata here in a list of dicts.
objs = files.get_container(self.auth.url,self.auth.token,self.container,None,None,None,None,None,True)
for obj in objs[1]: # For object in container data payload.
try:
bytes = obj['bytes']
last_modified = obj['last_modified']
hash = obj['hash']
name = obj['name'] # Defines marker for last object in list, and filename for usage.
marker = name
content_type = obj['content_type']
full_filelist.append({ 'name' : name, 'content_type' : content_type, 'last_modified' : last_modified , 'hash' : hash, 'bytes' : bytes })
except:
pass # Some of these objects are metadata.
return full_filelist
def listdir(self, abs_path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
files = []
dirs = []
for obj in container[1]:
if not obj['name'].startswith(abs_path):
continue
path = obj['name'][len(abs_path):].split('/')
key = path[0] if path[0] else path[1]
if not self.isdir(key):
files.append(key)
elif key not in dirs:
dirs.append(key)
return dirs, files
def listdir(self, path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
files = []
dirs = []
path = self.name_prefix + path
for obj in container[1]:
if not obj['name'].startswith(path):
continue
remaining_path = obj['name'][len(path):].split('/')
key = remaining_path[0] if remaining_path[0] else remaining_path[1]
if not self.isdir(key):
files.append(key)
elif key not in dirs:
dirs.append(key)
return dirs, files
def listdir(self, abs_path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
files = []
dirs = []
for obj in container[1]:
if not obj['name'].startswith(abs_path):
continue
path = obj['name'][len(abs_path):].split('/')
key = path[0] if path[0] else path[1]
if not self.isdir(key):
files.append(key)
elif key not in dirs:
dirs.append(key)
return dirs, files
def listdir(self, path):
container = swiftclient.get_container(self.storage_url, self.token,
self.container_name)
files = []
dirs = []
path = self.name_prefix + path
for obj in container[1]:
if not obj['name'].startswith(path):
continue
remaining_path = obj['name'][len(path):].split('/')
key = remaining_path[0] if remaining_path[0] else remaining_path[1]
if not self.isdir(key):
files.append(key)
elif key not in dirs:
dirs.append(key)
return dirs, files
delete_confirm = quote(params.get('delete_confirm', ''))
acl_edit = quote(params.get('acl_edit', ''))
meta_edit = quote(params.get('meta_edit', ''))
# whole container list
try:
whole_cont_list = self._get_whole_cont_list(storage_url, token)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
# whole object list, and object list for one page
try:
(cont_status,
_whole_obj_list) = get_container(storage_url,
token,
cont,
delimiter=self.delimiter,
prefix=prefix,
full_listing=True)
(cont_status, obj_list) = get_container(storage_url,
token,
cont,
limit=self.items_per_page,
delimiter=self.delimiter,
prefix=prefix,
marker=marker)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
whole_obj_list = zip(
[_o.get('name', _o.get('subdir')) for _o in _whole_obj_list], [
swiftclient.head_container("", dest_token, container_name, http_conn=dest_storage_cnx)
except (swiftclient.client.ClientException), e:
container_headers = orig_container_headers.copy()
for h in ("x-container-object-count", "x-trans-id", "x-container-bytes-used"):
del container_headers[h]
p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try:
swiftclient.put_container(url, dest_token, container_name, headers=container_headers)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason))
return
try:
dest_container_headers, dest_objects = swiftclient.get_container(
None, dest_token, container_name, http_conn=dest_storage_cnx
)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (container_name, e.http_reason))
return
do_headers = False
if len(dest_container_headers) != len(orig_container_headers):
do_headers = True
else:
for k, v in orig_container_headers.iteritems():
if k.startswith("x-container-meta") and k in dest_container_headers:
if dest_container_headers[k] != v:
do_headers = True
if do_headers:
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try:
swiftclient.put_container(url,
dest_token,
container_name,
headers=container_headers)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" %
(container_name, e.http_reason))
return
try:
dest_container_headers, dest_objects = swiftclient.get_container(
None,
dest_token,
container_name,
http_conn=dest_storage_cnx,
full_listing=True,
)
except (swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" %
(container_name, e.http_reason))
return
try:
header_key = 'x-container-meta-last-modified'
orig_ts = float(orig_container_headers[header_key])
dest_ts = float(dest_container_headers[header_key])
if orig_ts < dest_ts:
logging.info("Dest is up-to-date")
return
end_marker = params.get('end_marker', '')
prefix = params.get('prefix', '')
delete_confirm = quote(params.get('delete_confirm', ''))
acl_edit = quote(params.get('acl_edit', ''))
meta_edit = quote(params.get('meta_edit', ''))
# whole container list
try:
whole_cont_list = self._get_whole_cont_list(storage_url, token)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
# whole object list, and object list for one page
try:
(cont_status, _whole_obj_list) = get_container(storage_url, token, cont,
delimiter=self.delimiter,
prefix=prefix,
full_listing=True)
(cont_status, obj_list) = get_container(storage_url, token, cont,
limit=self.items_per_page,
delimiter=self.delimiter,
prefix=prefix, marker=marker)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
whole_obj_list = zip([_o.get('name',_o.get('subdir')) for _o in _whole_obj_list],
[unquote(_o.get('name', _o.get('subdir'))) for _o in _whole_obj_list])
obj_meta = {}
obj_unquote_name = {}
obj_delete_set_time = {}
path_type = len([i for i in [vrs, acc, cont, obj] if i])
# Nov2013: swift server does not set x-trans-id header
pass
p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try:
swiftclient.put_container(url,
dest_token, orig_container.name,
headers=container_headers)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (
orig_container.name, e.http_reason))
return
try:
dest_container_headers, dest_objects = swiftclient.get_container(
dest_storage_url, dest_token, orig_container.name, http_conn=dest_storage_cnx,
full_listing=True,
)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (
orig_container.name, e.http_reason))
return
try:
header_key = 'x-container-meta-last-modified'
orig_ts = float(container_headers[header_key])
dest_ts = float(dest_container_headers[header_key])
if orig_ts < dest_ts:
logging.info("Dest is up-to-date")
return
except(KeyError):
# last-modified swift middleware is not active
def list(self): return swiftclient.get_container(self.sturl, self.token, self.container)[1]
def get_cont(self, container_name): '''Get container info and list objects in container''' cont_data = SWIFT.get_container(self.swift_url, self.auth_token, container_name) return cont_data