def _get_whole_cont_list(self, storage_url, token):
""" whole container list """
(_junk, _whole_cont_list) = get_account(storage_url,
token,
full_listing=True)
return zip([_c['name'] for _c in _whole_cont_list],
[unquote(_c['name']) for _c in _whole_cont_list])
def search():
if 'url' in session and 'token' in session:
val = request.args.get('val')
print "#########################################", val
try:
output = swiftclient.get_account(url=session['url'], token=session['token'])
except swiftclient.ClientException:
return redirect(url_for('logout'))
print 'DEBUG: views.account().swiftclient.get_account: '+str(output)
account_info = output[0]
account_info['x-account-bytes-used'] = float(account_info['x-account-bytes-used']) /(1024**3)
cont_list = output[1]
cont_list_1 = []
for i in cont_list:
if val in i['name']:
cont_list_1.append(i)
acl_dic = {}
for i in cont_list_1:
acl_dic[i['name']] = Containers.check_read_acl(url=session['url'], token=session['token'], container_name=i['name'])
return render_template('account.html',
title='Account',
account_info=account_info,
cont_list=cont_list_1,
acl_dic=acl_dic)
else:
return redirect(url_for('login'))
def sync_account(self):
"""Sync a single account with url/tok to dest_url/dest_tok."""
orig_storage_cnx = self.get_cloudfiles_auth_orig()
dest_auth_url = get_config('auth', 'keystone_dest')
cfg = get_config('auth', 'keystone_dest_credentials')
(orig_user, orig_password, orig_tenant) = cfg.split(':')
dest_storage_url = get_config('auth', 'dest_storage_url')
# we assume orig and dest passwd are the same obv synchronized.
dst_st_url, dest_token = self.get_swift_auth(
dest_auth_url, orig_tenant,
orig_user, orig_password)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
try:
orig_containers = orig_storage_cnx.cloudfiles.get_all_containers()
dest_account_headers, dest_containers = (
swiftclient.get_account(None, dest_token,
http_conn=dest_storage_cnx,
full_listing=True))
except(swiftclient.client.ClientException), e:
logging.info("error getting account: %s" % (
e.http_reason))
return
def sync_account(self, orig_storage_url, orig_token, dest_storage_url, dest_token):
"""Sync a single account with url/tok to dest_url/dest_tok."""
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
account_id = os.path.basename(orig_storage_url.replace("AUTH_", ""))
try:
orig_account_headers, orig_containers = swiftclient.get_account(
None, orig_token, http_conn=orig_storage_cnx, full_listing=True
)
dest_account_headers, dest_containers = swiftclient.get_account(
None, dest_token, http_conn=dest_storage_cnx, full_listing=True
)
except (swiftclient.client.ClientException), e:
logging.info("error getting account: %s, %s" % (account_id, e.http_reason))
return
def list(url, token):
cont_list = swiftclient.get_account(url=url, token=token)
print type(cont_list)
return cont_list[1]
'''
def sync_account(self, orig_storage_url, orig_token,
dest_storage_url, dest_token):
"""Sync a single account with url/tok to dest_url/dest_tok."""
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
account_id = os.path.basename(orig_storage_url.replace("AUTH_", ''))
try:
orig_account_headers, orig_containers = (
swiftclient.get_account(None, orig_token,
http_conn=orig_storage_cnx,
full_listing=True))
dest_account_headers, dest_containers = (
swiftclient.get_account(None, dest_token,
http_conn=dest_storage_cnx,
full_listing=True))
except(swiftclient.client.ClientException), e:
logging.info("error getting account: %s, %s" % (
account_id, e.http_reason))
return
def test_one(self):
"""create 4 containers and verify that they are there"""
bucket = self.client['one'].create()
bucket2 = self.client['two'].create
bucket3 = self.client['three'].create()
bucket4 = self.client['four'].create()
self.auth_token = self.get_auth_token(congif.get('key'))
headers, containers = get_account(url, self.auth_token,
full_listing=True,
http_conn=self._get_conn(url))
self.assertEqual(len(containers), 4)
def index_account(self, account, verify=False):
"List all containers for the account and create task for each"
self.logger.debug("Indexing Account %s" % (account))
url = '/'.join(['http://127.0.0.1', 'v1', account])
headers, containers = get_account(url,
self.auth_token,
full_listing=True,
http_conn=self._get_conn(url))
self.logger.info("Found %s Container(s) in %s" %
(len(containers), account.encode('utf8')))
for container in containers:
self.indexer.publish_backfill_task('container',
account,
container['name'],
verify=verify)
def index_account(self):
verify=False
account_list =[config.get('account'), config.get('account2')]
"List all containers for the account and create task for each"
for account in account_list:
url = '/'.join(['http://127.0.0.1', 'v1', account])
headers, containers = get_account(url, self.auth_token,
full_listing=True,
http_conn=self._get_conn(url))
self.logger.info("Found %s Container(s) in %s" %
(len(containers), account.encode('utf8')))
for container in containers:
self.indexer.publish_backfill_task(
'container', account, container['name'], verify=verify)
def index_container_test(self, marker=None, verify=False):
# Index the container up to marker
url = mock.Mock()
headers = mock.Mock()
auth_token = mock.Mock()
conn = mock.Mock()
headers, containers = swiftclient.get_account(url,
auth_token,
full_listing=True,
http_conn=conn)
self.assertTrue(containers)
indexer = mock.Mock()
account = mock.Mock()
container = mock.Mock()
objects = mock.Mock()
headers = mock.Mock()
verify = mock.Mock()
task_complete = self.indexer.publish_backfill_task('container',
account,
container,
verify=verify)
self.assertTrue(task_complete)
update_handler = index.ContainerUpdateHandler(indexer,
account,
container,
index=index)
self.assertTrue(update_handler)
_id, props = update_handler.parse_props(headers)
self.assertTrue(_id)
task_complete = self.indexer.publish_backfill_task('object',
account,
objects,
verify=verify)
self.assertTrue(task_complete)
def index_account_test(self, verify=False):
"""Lists containers for the account and create backfill tasks."""
url = mock.Mock()
headers = mock.Mock()
auth_token = mock.Mock()
conn = mock.Mock()
headers, containers = swiftclient.get_account(url,
auth_token,
full_listing=True,
http_conn=conn)
self.assertTrue(containers)
account = mock.Mock()
verify = verify
task_complete = self.indexer.publish_backfill_task('container',
account,
container['name'],
verify=verify)
self.assertTrue(task_complete)
end_marker = params.get('end_marker', '')
delete_confirm = quote(params.get('delete_confirm', ''))
acl_edit = quote(params.get('acl_edit', ''))
meta_edit = quote(params.get('meta_edit', ''))
contsync_edit = quote(params.get('contsync_edit', ''))
# whole container list
try:
whole_cont_list = self._get_whole_cont_list(storage_url, token)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
# container list for one page
try:
(acct_status, cont_list) = get_account(storage_url,
token,
limit=self.items_per_page,
marker=marker)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
cont_meta = {}
cont_acl = {}
cont_unquote_name = {}
cont_version_cont = {}
cont_sync_to = {}
cont_sync_key = {}
# pick only one container for confiming and editing
edit_param = [acl_edit, delete_confirm, meta_edit, contsync_edit]
if any(edit_param):
edit_cont = filter(None, edit_param)[0]
def _get_whole_cont_list(self, storage_url, token):
""" whole container list """
(_junk, _whole_cont_list) = get_account(storage_url, token,
full_listing=True)
return zip([_c['name'] for _c in _whole_cont_list],
[unquote(_c['name']) for _c in _whole_cont_list])
end_marker = params.get('end_marker', '')
delete_confirm = quote(params.get('delete_confirm', ''))
acl_edit = quote(params.get('acl_edit', ''))
meta_edit = quote(params.get('meta_edit', ''))
contsync_edit = quote(params.get('contsync_edit', ''))
# whole container list
try:
whole_cont_list = self._get_whole_cont_list(storage_url, token)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
# container list for one page
try:
(acct_status, cont_list) = get_account(storage_url, token,
limit=self.items_per_page,
marker=marker)
except ClientException, err:
resp = Response(charset='utf8')
resp.status = err.http_status
return resp
cont_meta = {}
cont_acl = {}
cont_unquote_name = {}
cont_version_cont = {}
cont_sync_to = {}
cont_sync_key = {}
# pick only one container for confiming and editing
edit_param = [acl_edit, delete_confirm, meta_edit, contsync_edit]
if any(edit_param):
edit_cont = filter(None, edit_param)[0]
def agent():
ca.log("Starting!")
keystone = client.Client(token=ca.creds['token'], tenant_id=ca.creds['tenantId'],
auth_url=ca.creds['identity_url'])
catalog = keystone.service_catalog.catalog
text = ""
if ca.datastore:
old_usage = json.loads(ca.datastore)
else:
old_usage = AutoVivification()
usage = AutoVivification()
changes = AutoVivification()
object_csv_data = []
compute_csv_data = []
files = []
for entry in catalog['serviceCatalog']:
if entry['type'] == 'object-store':
text += "\nObject Storage\n--------------\n\n"
text += "%-15s %-25s %10s %10s %10s\n" % ("Region", "Container",
"Objects",
"Size",
"$/month")
for endpoint in entry['endpoints']:
headers, containers = swiftclient.get_account(endpoint['publicURL'],
ca.creds['token'],full_listing=True)
for container in containers:
try:
old_container = old_usage['object-store'][endpoint['region']][container['name']]
except ValueError:
old_container = dict
if old_container.get('count',0) != container['count'] or old_container.get('bytes',0) != container['bytes']:
if old_container.get('bytes',0) < container['bytes'] or old_container.get('count',0) < container['count']:
changes['object-store'][endpoint['region']][container['name']]['bytes'] = container['bytes'] - old_container.get('bytes',0)
changes['object-store'][endpoint['region']][container['name']]['count'] = container['count'] - old_container.get('count',0)
usage['object-store'][endpoint['region']][container['name']]['count'] = container['count']
usage['object-store'][endpoint['region']][container['name']]['bytes'] = container['bytes']
try:
old_usage['object-store'][endpoint['region']].pop(container['name'])
except ValueError:
pass
object_csv_data.append([endpoint['region'],container['name'],container['count'],container['bytes']])
text += "%-15s %-25s %10s %10s %10s\n" % (endpoint['region'], container['name'],
container['count'],
sizeof_fmt(container['bytes']),
"$"+storage_price(container['bytes']))
if entry['type'] == 'compute':
text += "\nCompute\n--------------\n\n"
text += "%-20s %-40s %7s %7s\n" % ("Region","Name", "Size", "$/month")
for endpoint in entry['endpoints']:
nova = novaclient.v1_1.client.Client(None,None,None,auth_url="")
nova.set_management_url(endpoint['publicURL'])
nova.client.auth_token = ca.creds['token']
flavors = dict()
for flavor in nova.flavors.list():
flavors[unicode(flavor.id)] = dict()
flavors[unicode(flavor.id)]["display"] = sizeof_fmt(flavor.ram*1048576)
flavors[unicode(flavor.id)]["price"] = get_compute_price(flavor.ram)
flavors[unicode(flavor.id)]["ram"] = flavor.ram
flavors[unicode(flavor.id)]["name"] = flavor.name
for server in nova.servers.list():
try:
oldserver = old_usage['compute'][endpoint['region']][str(server.id)]
except KeyError:
oldserver = None
if not oldserver:
changes['compute'][endpoint['region']][server.id]['size'] = flavors[server.flavor['id']]["ram"]
changes['compute'][endpoint['region']][server.id]['name'] = server.name
changes['compute'][endpoint['region']][server.id]['added'] = True
else:
try:
old_usage['compute'][endpoint['region']].pop(str(server.id))
except ValueError:
pass
compute_csv_data.append([endpoint['region'],str(server.id),str(server.name),flavors[server.flavor['id']]["name"],flavors[server.flavor['id']]["ram"]])
usage['compute'][endpoint['region']][server.id]['size'] = flavors[server.flavor['id']]["ram"]
usage['compute'][endpoint['region']][server.id]['name'] = server.name
text += "%-20s %-40s %7s %7s\n" % (endpoint['region'], server.name,
flavors[server.flavor['id']]["display"],
"$"+compute_price(flavors[server.flavor['id']]["price"]))
if object_csv_data:
object_csv = StringIO.StringIO()
object_csv_writer = csv.writer(object_csv)
object_csv_writer.writerows([["region","container","objects","bytes"]])
object_csv_writer.writerows(object_csv_data)
csv_data = object_csv.getvalue()
files.append({"file-name":"object-storage.csv","contents":csv_data})
if compute_csv_data:
compute_csv = StringIO.StringIO()
compute_csv_writer = csv.writer(compute_csv)
compute_csv_writer.writerows([["region","id","name","flavor","ram"]])
compute_csv_writer.writerows(compute_csv_data)
csv_data = compute_csv.getvalue()
files.append({"file-name":"compute.csv","contents":csv_data})
#print text
ca.store(json.dumps(usage))
changes_html = ""
for dataset in [changes, old_usage]:
for service, service_data in dataset.items():
for zone, zone_data in service_data.items():
for change, change_data in zone_data.items():
if service == 'object-store':
changes_html += "<tr><td>"+str(service)+"</td>"
changes_html += "<td>"+str(zone)+"</td>"
changes_html += "<td>"+str(change)+"</td><td>"
if change_data.get("count"):
if change_data['count'] > 0:
changes_html += "<font color='green'>+"+str(change_data['count'])+" obj</font>"
elif change_data['count'] < 0:
changes_html += "<font color='red'>-"+str(change_data['count'])+" obj</font>"
changes_html += "</td><td>"
if change_data.get("bytes"):
if change_data['bytes'] > 0:
changes_html += "<font color='green'>+"+sizeof_fmt(change_data['count'])+"</font>"
elif change_data['bytes'] < 0:
changes_html += "<font color='red'>-"+sizeof_fmt(change_data['count'])+"</font>"
changes_html += "</td></tr>\n"
elif service == 'compute':
if change_data.get('name'):
changes_html += "<tr><td>"+str(service)+"</td>"
changes_html += "<td>"+str(zone)+"</td>"
changes_html += "<td>"+str(change_data.get('name'))+"</td><td colspan=2>"
if change_data.get('added'):
changes_html += "<font color='green'>Created "+sizeof_fmt(change_data['size']*1048576)+" server</font>"
else:
changes_html += "<font color='red'>Deleted "+sizeof_fmt(change_data['size']*1048576)+" server</font>"
changes_html += "</td></tr>\n"
if changes_html != '':
changes_html = """
<table>
<tr>
<td><b>Service</b></td>
<td><b>Zone</b></td>
<td><b>Item</b></td>
<td colspan=2><b>Change</b></td>
</tr>
""" + changes_html + """
</table>
"""
ca.email("Usage Change",text, changes_html, files)
def get_acct(self): acct_data = SWIFT.get_account(self.swift_url, self.auth_token) return acct_data
