class Main(object):
def __init__(self):
parser = OptionParser(
usage="%prog [options] memstore_admin password",
version="%prog v" + VERSION,
description=
"List Miniserver snapshots stored in a Memstore(tm) instance.",
epilog=
"For further information, please visit: http://www.memset.com/cloud/storage/"
)
parser.add_option(
"--auth_url",
type="str",
dest="auth_url",
default=AUTH_URL,
help="Auth URL (default: %s)" % AUTH_URL,
)
self.options, self.args = parser.parse_args()
if len(self.args) != 2:
parser.error("Not enough parameters provided")
self.conn = Connection(self.options.auth_url, self.args[0],
self.args[1])
def run(self):
info, snapshots = self.conn.get_container('miniserver-snapshots',
delimiter='/')
for snap in snapshots:
if 'subdir' not in snap.keys():
continue
_, objects = self.conn.get_container('miniserver-snapshots',
prefix=snap["subdir"])
snap["bytes-used"] = 0
snap["parts"] = 0
readme = False
for obj in objects:
snap["bytes-used"] += obj["bytes"]
if '.part' in obj.get("name", ""):
snap["parts"] += 1
elif obj.get("name", "").lower().endswith("readme.txt"):
readme = True
snap["date"] = obj["last_modified"]
if not readme:
snap[
"notes"] = "README.txt not found, it may be an incomplete snapshot"
result = dict((key[len("x-container-"):], value)
for key, value in info.items()
if key.startswith("x-container-"))
result["snapshots"] = snapshots
print json.dumps(result, indent=1)
class Main(object):
def __init__(self):
parser = OptionParser(
usage="%prog [options] memstore_admin password",
version="%prog v" + VERSION,
description="List Miniserver snapshots stored in a Memstore(tm) instance.",
epilog="For further information, please visit: http://www.memset.com/cloud/storage/",
)
parser.add_option(
"--auth_url", type="str", dest="auth_url", default=AUTH_URL, help="Auth URL (default: %s)" % AUTH_URL
)
self.options, self.args = parser.parse_args()
if len(self.args) != 2:
parser.error("Not enough parameters provided")
self.conn = Connection(self.options.auth_url, self.args[0], self.args[1])
def run(self):
info, snapshots = self.conn.get_container("miniserver-snapshots", delimiter="/")
for snap in snapshots:
if "subdir" not in snap.keys():
continue
_, objects = self.conn.get_container("miniserver-snapshots", prefix=snap["subdir"])
snap["bytes-used"] = 0
snap["parts"] = 0
readme = False
for obj in objects:
snap["bytes-used"] += obj["bytes"]
if ".part" in obj.get("name", ""):
snap["parts"] += 1
elif obj.get("name", "").lower().endswith("readme.txt"):
readme = True
snap["date"] = obj["last_modified"]
if not readme:
snap["notes"] = "README.txt not found, it may be an incomplete snapshot"
result = dict(
(key[len("x-container-") :], value) for key, value in info.items() if key.startswith("x-container-")
)
result["snapshots"] = snapshots
print json.dumps(result, indent=1)
def main(args):
output_bucket = args.get("output_bucket")
mapper_bucket = args["mapper_bucket"]
input_bucket = args["input_bucket"]
n_mapper = args["n_mapper"]
_authurl = "http://"+args.get('url')+":8080/auth/v1.0"
conn = Connection(
authurl=_authurl,
user=_user,
key=_key,
tenant_name=_tenant_name,
auth_version=_auth_version
)
# Fetch all the keys
all_keys = []
for obj in conn.get_container(input_bucket)[1]:
all_keys.append(obj['name'])
total_size = len(all_keys)
batch_size = 0
if total_size % n_mapper == 0:
batch_size = total_size / n_mapper
else:
batch_size = total_size // n_mapper + 1
for idx in range(n_mapper):
print("mapper-" + str(idx) + ":" + str(all_keys[idx * batch_size: (idx + 1) * batch_size]))
pool = ThreadPool(n_mapper)
invoke_mapper_partial = partial(invoke_mapper_action, "default/mapper", mapper_bucket, input_bucket, all_keys, batch_size)
pool.map(invoke_mapper_partial, range(n_mapper))
pool.close()
pool.join()
while True:
res_s3 = conn.get_container(mapper_bucket)[1]
if "Contents" not in res_s3.keys(): job_keys = []
else: job_keys = res_s3["Contents"]
print("Wait Mapper Jobs ...")
time.sleep(5)
if len(job_keys) == n_mapper:
print("[*] Map Done : mapper " + str(len(job_keys)) + " finished.")
break
return {"input_reducer_bucket": mapper_bucket, "output_bucket": output_bucket}
def main(args):
output_bucket = args.get("output_bucket")
input_reducer_bucket = args.get("input_reducer_bucket")
_authurl = "http://" + args.get('url') + ":8080/auth/v1.0"
conn = Connection(authurl=_authurl,
user=_user,
key=_key,
tenant_name=_tenant_name,
auth_version=_auth_version)
all_keys = []
for obj in conn.get_container(input_reducer_bucket)[1]:
all_keys.append(obj['name'])
final_str = []
for key in all_keys:
start = time()
response = conn.get_object(input_reducer_bucket, key)
contents = response.read()
contents = contents.decode()
final_str.append(to_string(getResultReduce(contents)))
final_str = "\n".join(final_str)
print(final_str)
final_res = getResultReduce(final_str)
conn.put_object(output_bucket,
"reduce_res.json",
contents=json.dumps(final_res))
return {"res": "good"}
class SwiftStorage(object):
def __init__(self, account, url, user, key):
self.conn = Connection(authurl=url, user=user, key=key)
self.cont = self.conn.get_container(account)
self.account = account
def list(self):
return [x["name"] for x in self.cont[1]]
def get_attrs(self, uuid):
attrs = {}
k = self.conn.get_object(container=self.account, obj=uuid)[0]
print k
attrs['tags'] = json.loads(k['x-object-meta-tags'])
attrs['flags'] = json.loads(k['x-object-meta-flags'])
attrs['stored'] = int(k['x-object-meta-stored'])
return attrs
def get_message(self, uuid):
return self.conn.get_object(container=self.account, obj=uuid)[1]
def put_message(self, uuid, msg, attrs):
self.conn.put_object(container=self.account, obj=uuid, contents=msg, headers=attrs)
self.put_attrs(uuid, attrs)
def put_attrs(self, uuid, attrs):
newattrs = {'X-Object-Meta-Tags': json.dumps(attrs['tags']),
'X-Object-Meta-Flags': json.dumps(attrs['flags']),
'X-Object-Meta-Stored': str(attrs['stored'])}
print newattrs
self.conn.post_object(container=self.account, obj=uuid, headers=newattrs)
def del_message(self, uuid):
self.conn.delete_object(container=self.account, obj=uuid)
def run(self, terms, variables=None, **kwargs):
"""Returns Ironic Inspector introspection data.
Access swift and return introspection data for all nodes.
:returns a list of tuples, one for each node.
"""
ret = []
session = get_auth_session(kwargs.get('auth_url'), "ironic", "service",
kwargs.get('password'))
swift_client = Connection(session=session)
container = swift_client.get_container("ironic-inspector")
for item in container[1]:
if item['name'].startswith('inspector_data') and \
not item['name'].endswith("UNPROCESSED"):
obj = swift_client.get_object("ironic-inspector", item['name'])
ret.append((item['name'], obj))
return ret
class Swift(object):
def __init__(self, bucket, noop):
"""Setup the S3 storage backend with the bucket we will use and
optional region."""
# This is our Swift container
self.bucket = bucket
self.noop = noop
# We assume your environment variables are set correctly just like
# you would for the swift command line util
try:
self.conn = Connection(authurl=os.environ["ST_AUTH"],
user=os.environ["ST_USER"],
key=os.environ["ST_KEY"],
timeout=30)
except KeyError:
logger.warning(
"Missing environment variables for Swift authentication")
logger.warning("Bailing...")
sys.exit(1)
headers, objs = self.conn.get_account(self.bucket)
for i in objs:
logger.debug("Searching for bucket %s == %s" % (self.bucket, i))
if not noop and self.bucket not in objs:
self.conn.put_container(self.bucket)
def list(self, prefix=None):
"""Return all keys in this bucket."""
headers, objs = self.conn.get_container(self.bucket, prefix=prefix)
while objs:
# Handle paging
i = {}
for i in objs:
yield i["name"]
headers, objs = self.conn.get_container(self.bucket,
marker=i["name"],
prefix=prefix)
def get(self, src):
"""Return the contents of src from S3 as a string."""
try:
headers, obj = self.conn.get_object(self.bucket, src)
return obj
except ClientException:
# Request failed....object doesn't exist
return None
def put(self, dst, data):
"""Store the contents of the string data at a key named by dst
in S3."""
if self.noop:
logger.info("No-Op Put: %s" % dst)
else:
self.conn.put_object(self.bucket, dst, data)
def delete(self, src):
"""Delete the object in S3 referenced by the key name src."""
if self.noop:
logger.info("No-Op Delete: %s" % src)
else:
self.conn.delete_object(self.bucket, src)
class OpenstackDriver(BaseDriver):
"""OpenStackDriver for Object Storage"""
def __init__(self, cloud_config):
super(OpenstackDriver, self).__init__()
self.auth_url = cloud_config['os_auth_url']
self.project_name = cloud_config['os_project_name']
self.username = cloud_config['os_username']
self.password = cloud_config['os_password']
self.user_domain_name = \
cloud_config.get('os_project_domain_name', 'default')
self.project_domain_name = \
cloud_config.get('os_user_domain_name', 'default')
self.driver_name = \
cloud_config.get('driver_name', 'default')
self.tenant_id = cloud_config.get('tenant_id', None)
self.limit = cloud_config.get('limit', None)
self.auth_version = \
cloud_config.get('os_auth_version', '2')
self._setup()
def _setup(self):
auth = v3.Password(auth_url=self.auth_url,
user_domain_name=self.user_domain_name,
username=self.username,
password=self.password,
project_domain_name=self.project_domain_name,
project_name=self.project_name)
sess = session.Session(auth=auth)
self.client = Connection(auth_version=self.auth_version, session=sess)
self.quota = OpenStackQuota(
self.client, self.tenant_id, self.limit)
def create_container(self, container, **kwargs):
return self.client.put_container(container)
def delete_container(self, container):
return self.client.delete_container(container)
def list_containers(self):
return self.client.get_account()[1]
def stat_container(self, container):
return self.client.head_container(container)
def update_container(self, container, metadata, **kwargs):
metadata = {('x-container-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-container-meta-')}
return self.client.post_container(container, metadata, **kwargs)
def upload_object(self, container, obj, contents,
content_length=None, metadata=None, **kwargs):
if metadata:
metadata = {('x-object-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-object-meta-')}
return self.client.put_object(container, obj, contents=contents,
content_length=content_length,
headers=metadata, **kwargs)
def download_object(self, container, obj, **kwargs):
return self.client.get_object(container, obj, **kwargs)
def stat_object(self, container, obj):
return self.client.head_object(container, obj)
def delete_object(self, container, obj, **kwargs):
return self.client.delete_object(container, obj, **kwargs)
def list_container_objects(self, container, prefix=None, delimiter=None):
return self.client.get_container(container, prefix, delimiter)[1]
def update_object(self, container, obj, metadata, **kwargs):
# Format metedata key
metadata = {('x-object-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-object-meta-')}
return self.client.post_object(container, obj, metadata, **kwargs)
def copy_object(self, container, obj, metadata=None,
destination=None, **kwargs):
return self.client.copy_object(container, obj, headers=metadata,
destination=destination, **kwargs)
class OpenstackDriver(BaseDriver):
"""OpenStackDriver for Object Storage"""
def __init__(self, cloud_config):
super(OpenstackDriver, self).__init__()
self.auth_url = cloud_config['os_auth_url']
self.project_name = cloud_config['os_project_name']
self.username = cloud_config['os_username']
self.password = cloud_config['os_password']
self.user_domain_name = \
cloud_config.get('os_project_domain_name', 'default')
self.project_domain_name = \
cloud_config.get('os_user_domain_name', 'default')
self.driver_name = \
cloud_config.get('driver_name', 'default')
self.tenant_id = cloud_config.get('tenant_id', None)
self.limit = cloud_config.get('limit', None)
self.auth_version = \
cloud_config.get('os_auth_version', '2')
self._setup()
def _setup(self):
auth = v3.Password(auth_url=self.auth_url,
user_domain_name=self.user_domain_name,
username=self.username,
password=self.password,
project_domain_name=self.project_domain_name,
project_name=self.project_name)
sess = session.Session(auth=auth)
self.client = Connection(auth_version=self.auth_version, session=sess)
self.quota = OpenStackQuota(self.client, self.tenant_id, self.limit)
def create_container(self, container, **kwargs):
return self.client.put_container(container)
def delete_container(self, container):
return self.client.delete_container(container)
def list_containers(self):
return self.client.get_account()[1]
def stat_container(self, container):
return self.client.head_container(container)
def update_container(self, container, metadata, **kwargs):
metadata = {('x-container-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-container-meta-')}
return self.client.post_container(container, metadata, **kwargs)
def upload_object(self,
container,
obj,
contents,
content_length=None,
metadata=None,
**kwargs):
if metadata:
metadata = {('x-object-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-object-meta-')}
return self.client.put_object(container,
obj,
contents=contents,
content_length=content_length,
headers=metadata,
**kwargs)
def download_object(self, container, obj, **kwargs):
return self.client.get_object(container, obj, **kwargs)
def stat_object(self, container, obj):
return self.client.head_object(container, obj)
def delete_object(self, container, obj, **kwargs):
return self.client.delete_object(container, obj, **kwargs)
def list_container_objects(self, container, prefix=None, delimiter=None):
return self.client.get_container(container, prefix, delimiter)[1]
def update_object(self, container, obj, metadata, **kwargs):
# Format metedata key
metadata = {('x-object-meta-' + key.strip()): value
for key, value in metadata.items()
if not key.strip().startswith('x-object-meta-')}
return self.client.post_object(container, obj, metadata, **kwargs)
def copy_object(self,
container,
obj,
metadata=None,
destination=None,
**kwargs):
return self.client.copy_object(container,
obj,
headers=metadata,
destination=destination,
**kwargs)
class ObjectStore:
RESP_LIMIT = 10000 # serverside limit of the response
def __init__(self, container, name):
if name == 'bag_brk':
OBJECTSTORE['user'] = '******'
OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD
OBJECTSTORE['os_options'][
'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18'
OBJECTSTORE['tenant_name'] = 'BGE000081_BAG'
else:
OBJECTSTORE['user'] = '******'
OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD
OBJECTSTORE['os_options'][
'tenant_id'] = '1776010a62684386a08b094d89ce08d9'
OBJECTSTORE['tenant_name'] = 'BGE000081_BGT'
self.conn = Connection(**OBJECTSTORE)
self.container = container
def get_store_object(self, name):
"""
Returns the object store
:param name:
:return:
"""
return self.conn.get_object(self.container, name)[1]
def get_store_objects(self, path):
return self._get_full_container_list([], prefix=path)
def _get_full_container_list(self, seed, **kwargs):
kwargs['limit'] = self.RESP_LIMIT
if len(seed):
kwargs['marker'] = seed[-1]['name']
_, page = self.conn.get_container(self.container, **kwargs)
seed.extend(page)
return seed if len(page) < self.RESP_LIMIT else \
self._get_full_container_list(seed, **kwargs)
def folders(self, path):
objects_from_store = self._get_full_container_list(
[], delimiter='/', prefix=path)
return [
store_object['subdir'] for store_object in objects_from_store if
'subdir' in store_object]
def files(self, path, file_id):
file_list = self._get_full_container_list(
[], delimiter='/', prefix=path + file_id)
for file_object in file_list:
file_object['container'] = self.container
return file_list
def put_to_objectstore(self, object_name, object_content, content_type):
return self.conn.put_object(self.container, object_name,
contents=object_content,
content_type=content_type)
def delete_from_objectstore(self, object_name):
return self.conn.delete_object(self.container, object_name)
class Swift(object):
def __init__(self, bucket, noop):
"""Setup the S3 storage backend with the bucket we will use and
optional region."""
# This is our Swift container
self.bucket = bucket
self.noop = noop
# We assume your environment variables are set correctly just like
# you would for the swift command line util
try:
self.conn = Connection(authurl=os.environ["ST_AUTH"],
user=os.environ["ST_USER"],
key=os.environ["ST_KEY"])
except KeyError:
logger.warning("Missing environment variables for Swift authentication")
logger.warning("Bailing...")
sys.exit(1)
headers, objs = self.conn.get_account(self.bucket)
for i in objs:
logger.debug("Searching for bucket %s == %s" % (self.bucket, i))
if not noop and self.bucket not in objs:
self.conn.put_container(self.bucket)
def list(self, prefix=None):
"""Return all keys in this bucket."""
headers, objs = self.conn.get_container(self.bucket, prefix=prefix)
while objs:
# Handle paging
i = {}
for i in objs:
yield i["name"]
headers, objs = self.conn.get_container(self.bucket,
marker=i["name"], prefix=prefix)
def get(self, src):
"""Return the contents of src from S3 as a string."""
try:
headers, obj = self.conn.get_object(self.bucket, src)
return obj
except ClientException:
# Request failed....object doesn't exist
return None
def put(self, dst, data):
"""Store the contents of the string data at a key named by dst
in S3."""
if self.noop:
logger.info("No-Op Put: %s" % dst)
else:
self.conn.put_object(self.bucket, dst, data)
def delete(self, src):
"""Delete the object in S3 referenced by the key name src."""
if self.noop:
logger.info("No-Op Delete: %s" % src)
else:
self.conn.delete_object(self.bucket, src)
class ObjectStore:
RESP_LIMIT = 10000 # serverside limit of the response
def __init__(self, container, name):
if name == 'bag_brk':
OBJECTSTORE['user'] = '******'
OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD
OBJECTSTORE['os_options'][
'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18'
OBJECTSTORE['tenant_name'] = 'BGE000081_BAG'
else:
OBJECTSTORE['user'] = '******'
OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD
OBJECTSTORE['os_options'][
'tenant_id'] = '1776010a62684386a08b094d89ce08d9'
OBJECTSTORE['tenant_name'] = 'BGE000081_BGT'
self.conn = Connection(**OBJECTSTORE)
self.container = container
def get_store_object(self, name):
"""
Returns the object store
:param name:
:return:
"""
return self.conn.get_object(self.container, name)[1]
def get_store_objects(self, path):
return self._get_full_container_list([], prefix=path)
def _get_full_container_list(self, seed, **kwargs):
kwargs['limit'] = self.RESP_LIMIT
if len(seed):
kwargs['marker'] = seed[-1]['name']
_, page = self.conn.get_container(self.container, **kwargs)
seed.extend(page)
return seed if len(page) < self.RESP_LIMIT else \
self._get_full_container_list(seed, **kwargs)
def folders(self, path):
objects_from_store = self._get_full_container_list([],
delimiter='/',
prefix=path)
return [
store_object['subdir'] for store_object in objects_from_store
if 'subdir' in store_object
]
def files(self, path, file_id):
file_list = self._get_full_container_list([],
delimiter='/',
prefix=path + file_id)
for file_object in file_list:
file_object['container'] = self.container
return file_list
def put_to_objectstore(self, object_name, object_content, content_type):
return self.conn.put_object(self.container,
object_name,
contents=object_content,
content_type=content_type)
def delete_from_objectstore(self, object_name):
return self.conn.delete_object(self.container, object_name)