class Main(object): def __init__(self): parser = OptionParser( usage="%prog [options] memstore_admin password", version="%prog v" + VERSION, description= "List Miniserver snapshots stored in a Memstore(tm) instance.", epilog= "For further information, please visit: http://www.memset.com/cloud/storage/" ) parser.add_option( "--auth_url", type="str", dest="auth_url", default=AUTH_URL, help="Auth URL (default: %s)" % AUTH_URL, ) self.options, self.args = parser.parse_args() if len(self.args) != 2: parser.error("Not enough parameters provided") self.conn = Connection(self.options.auth_url, self.args[0], self.args[1]) def run(self): info, snapshots = self.conn.get_container('miniserver-snapshots', delimiter='/') for snap in snapshots: if 'subdir' not in snap.keys(): continue _, objects = self.conn.get_container('miniserver-snapshots', prefix=snap["subdir"]) snap["bytes-used"] = 0 snap["parts"] = 0 readme = False for obj in objects: snap["bytes-used"] += obj["bytes"] if '.part' in obj.get("name", ""): snap["parts"] += 1 elif obj.get("name", "").lower().endswith("readme.txt"): readme = True snap["date"] = obj["last_modified"] if not readme: snap[ "notes"] = "README.txt not found, it may be an incomplete snapshot" result = dict((key[len("x-container-"):], value) for key, value in info.items() if key.startswith("x-container-")) result["snapshots"] = snapshots print json.dumps(result, indent=1)
class Main(object): def __init__(self): parser = OptionParser( usage="%prog [options] memstore_admin password", version="%prog v" + VERSION, description="List Miniserver snapshots stored in a Memstore(tm) instance.", epilog="For further information, please visit: http://www.memset.com/cloud/storage/", ) parser.add_option( "--auth_url", type="str", dest="auth_url", default=AUTH_URL, help="Auth URL (default: %s)" % AUTH_URL ) self.options, self.args = parser.parse_args() if len(self.args) != 2: parser.error("Not enough parameters provided") self.conn = Connection(self.options.auth_url, self.args[0], self.args[1]) def run(self): info, snapshots = self.conn.get_container("miniserver-snapshots", delimiter="/") for snap in snapshots: if "subdir" not in snap.keys(): continue _, objects = self.conn.get_container("miniserver-snapshots", prefix=snap["subdir"]) snap["bytes-used"] = 0 snap["parts"] = 0 readme = False for obj in objects: snap["bytes-used"] += obj["bytes"] if ".part" in obj.get("name", ""): snap["parts"] += 1 elif obj.get("name", "").lower().endswith("readme.txt"): readme = True snap["date"] = obj["last_modified"] if not readme: snap["notes"] = "README.txt not found, it may be an incomplete snapshot" result = dict( (key[len("x-container-") :], value) for key, value in info.items() if key.startswith("x-container-") ) result["snapshots"] = snapshots print json.dumps(result, indent=1)
def main(args): output_bucket = args.get("output_bucket") mapper_bucket = args["mapper_bucket"] input_bucket = args["input_bucket"] n_mapper = args["n_mapper"] _authurl = "http://"+args.get('url')+":8080/auth/v1.0" conn = Connection( authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version ) # Fetch all the keys all_keys = [] for obj in conn.get_container(input_bucket)[1]: all_keys.append(obj['name']) total_size = len(all_keys) batch_size = 0 if total_size % n_mapper == 0: batch_size = total_size / n_mapper else: batch_size = total_size // n_mapper + 1 for idx in range(n_mapper): print("mapper-" + str(idx) + ":" + str(all_keys[idx * batch_size: (idx + 1) * batch_size])) pool = ThreadPool(n_mapper) invoke_mapper_partial = partial(invoke_mapper_action, "default/mapper", mapper_bucket, input_bucket, all_keys, batch_size) pool.map(invoke_mapper_partial, range(n_mapper)) pool.close() pool.join() while True: res_s3 = conn.get_container(mapper_bucket)[1] if "Contents" not in res_s3.keys(): job_keys = [] else: job_keys = res_s3["Contents"] print("Wait Mapper Jobs ...") time.sleep(5) if len(job_keys) == n_mapper: print("[*] Map Done : mapper " + str(len(job_keys)) + " finished.") break return {"input_reducer_bucket": mapper_bucket, "output_bucket": output_bucket}
def main(args): output_bucket = args.get("output_bucket") input_reducer_bucket = args.get("input_reducer_bucket") _authurl = "http://" + args.get('url') + ":8080/auth/v1.0" conn = Connection(authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version) all_keys = [] for obj in conn.get_container(input_reducer_bucket)[1]: all_keys.append(obj['name']) final_str = [] for key in all_keys: start = time() response = conn.get_object(input_reducer_bucket, key) contents = response.read() contents = contents.decode() final_str.append(to_string(getResultReduce(contents))) final_str = "\n".join(final_str) print(final_str) final_res = getResultReduce(final_str) conn.put_object(output_bucket, "reduce_res.json", contents=json.dumps(final_res)) return {"res": "good"}
class SwiftStorage(object): def __init__(self, account, url, user, key): self.conn = Connection(authurl=url, user=user, key=key) self.cont = self.conn.get_container(account) self.account = account def list(self): return [x["name"] for x in self.cont[1]] def get_attrs(self, uuid): attrs = {} k = self.conn.get_object(container=self.account, obj=uuid)[0] print k attrs['tags'] = json.loads(k['x-object-meta-tags']) attrs['flags'] = json.loads(k['x-object-meta-flags']) attrs['stored'] = int(k['x-object-meta-stored']) return attrs def get_message(self, uuid): return self.conn.get_object(container=self.account, obj=uuid)[1] def put_message(self, uuid, msg, attrs): self.conn.put_object(container=self.account, obj=uuid, contents=msg, headers=attrs) self.put_attrs(uuid, attrs) def put_attrs(self, uuid, attrs): newattrs = {'X-Object-Meta-Tags': json.dumps(attrs['tags']), 'X-Object-Meta-Flags': json.dumps(attrs['flags']), 'X-Object-Meta-Stored': str(attrs['stored'])} print newattrs self.conn.post_object(container=self.account, obj=uuid, headers=newattrs) def del_message(self, uuid): self.conn.delete_object(container=self.account, obj=uuid)
def run(self, terms, variables=None, **kwargs): """Returns Ironic Inspector introspection data. Access swift and return introspection data for all nodes. :returns a list of tuples, one for each node. """ ret = [] session = get_auth_session(kwargs.get('auth_url'), "ironic", "service", kwargs.get('password')) swift_client = Connection(session=session) container = swift_client.get_container("ironic-inspector") for item in container[1]: if item['name'].startswith('inspector_data') and \ not item['name'].endswith("UNPROCESSED"): obj = swift_client.get_object("ironic-inspector", item['name']) ret.append((item['name'], obj)) return ret
class Swift(object): def __init__(self, bucket, noop): """Setup the S3 storage backend with the bucket we will use and optional region.""" # This is our Swift container self.bucket = bucket self.noop = noop # We assume your environment variables are set correctly just like # you would for the swift command line util try: self.conn = Connection(authurl=os.environ["ST_AUTH"], user=os.environ["ST_USER"], key=os.environ["ST_KEY"], timeout=30) except KeyError: logger.warning( "Missing environment variables for Swift authentication") logger.warning("Bailing...") sys.exit(1) headers, objs = self.conn.get_account(self.bucket) for i in objs: logger.debug("Searching for bucket %s == %s" % (self.bucket, i)) if not noop and self.bucket not in objs: self.conn.put_container(self.bucket) def list(self, prefix=None): """Return all keys in this bucket.""" headers, objs = self.conn.get_container(self.bucket, prefix=prefix) while objs: # Handle paging i = {} for i in objs: yield i["name"] headers, objs = self.conn.get_container(self.bucket, marker=i["name"], prefix=prefix) def get(self, src): """Return the contents of src from S3 as a string.""" try: headers, obj = self.conn.get_object(self.bucket, src) return obj except ClientException: # Request failed....object doesn't exist return None def put(self, dst, data): """Store the contents of the string data at a key named by dst in S3.""" if self.noop: logger.info("No-Op Put: %s" % dst) else: self.conn.put_object(self.bucket, dst, data) def delete(self, src): """Delete the object in S3 referenced by the key name src.""" if self.noop: logger.info("No-Op Delete: %s" % src) else: self.conn.delete_object(self.bucket, src)
class OpenstackDriver(BaseDriver): """OpenStackDriver for Object Storage""" def __init__(self, cloud_config): super(OpenstackDriver, self).__init__() self.auth_url = cloud_config['os_auth_url'] self.project_name = cloud_config['os_project_name'] self.username = cloud_config['os_username'] self.password = cloud_config['os_password'] self.user_domain_name = \ cloud_config.get('os_project_domain_name', 'default') self.project_domain_name = \ cloud_config.get('os_user_domain_name', 'default') self.driver_name = \ cloud_config.get('driver_name', 'default') self.tenant_id = cloud_config.get('tenant_id', None) self.limit = cloud_config.get('limit', None) self.auth_version = \ cloud_config.get('os_auth_version', '2') self._setup() def _setup(self): auth = v3.Password(auth_url=self.auth_url, user_domain_name=self.user_domain_name, username=self.username, password=self.password, project_domain_name=self.project_domain_name, project_name=self.project_name) sess = session.Session(auth=auth) self.client = Connection(auth_version=self.auth_version, session=sess) self.quota = OpenStackQuota( self.client, self.tenant_id, self.limit) def create_container(self, container, **kwargs): return self.client.put_container(container) def delete_container(self, container): return self.client.delete_container(container) def list_containers(self): return self.client.get_account()[1] def stat_container(self, container): return self.client.head_container(container) def update_container(self, container, metadata, **kwargs): metadata = {('x-container-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-container-meta-')} return self.client.post_container(container, metadata, **kwargs) def upload_object(self, container, obj, contents, content_length=None, metadata=None, **kwargs): if metadata: metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.put_object(container, obj, contents=contents, content_length=content_length, headers=metadata, **kwargs) def download_object(self, container, obj, **kwargs): return self.client.get_object(container, obj, **kwargs) def stat_object(self, container, obj): return self.client.head_object(container, obj) def delete_object(self, container, obj, **kwargs): return self.client.delete_object(container, obj, **kwargs) def list_container_objects(self, container, prefix=None, delimiter=None): return self.client.get_container(container, prefix, delimiter)[1] def update_object(self, container, obj, metadata, **kwargs): # Format metedata key metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.post_object(container, obj, metadata, **kwargs) def copy_object(self, container, obj, metadata=None, destination=None, **kwargs): return self.client.copy_object(container, obj, headers=metadata, destination=destination, **kwargs)
class OpenstackDriver(BaseDriver): """OpenStackDriver for Object Storage""" def __init__(self, cloud_config): super(OpenstackDriver, self).__init__() self.auth_url = cloud_config['os_auth_url'] self.project_name = cloud_config['os_project_name'] self.username = cloud_config['os_username'] self.password = cloud_config['os_password'] self.user_domain_name = \ cloud_config.get('os_project_domain_name', 'default') self.project_domain_name = \ cloud_config.get('os_user_domain_name', 'default') self.driver_name = \ cloud_config.get('driver_name', 'default') self.tenant_id = cloud_config.get('tenant_id', None) self.limit = cloud_config.get('limit', None) self.auth_version = \ cloud_config.get('os_auth_version', '2') self._setup() def _setup(self): auth = v3.Password(auth_url=self.auth_url, user_domain_name=self.user_domain_name, username=self.username, password=self.password, project_domain_name=self.project_domain_name, project_name=self.project_name) sess = session.Session(auth=auth) self.client = Connection(auth_version=self.auth_version, session=sess) self.quota = OpenStackQuota(self.client, self.tenant_id, self.limit) def create_container(self, container, **kwargs): return self.client.put_container(container) def delete_container(self, container): return self.client.delete_container(container) def list_containers(self): return self.client.get_account()[1] def stat_container(self, container): return self.client.head_container(container) def update_container(self, container, metadata, **kwargs): metadata = {('x-container-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-container-meta-')} return self.client.post_container(container, metadata, **kwargs) def upload_object(self, container, obj, contents, content_length=None, metadata=None, **kwargs): if metadata: metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.put_object(container, obj, contents=contents, content_length=content_length, headers=metadata, **kwargs) def download_object(self, container, obj, **kwargs): return self.client.get_object(container, obj, **kwargs) def stat_object(self, container, obj): return self.client.head_object(container, obj) def delete_object(self, container, obj, **kwargs): return self.client.delete_object(container, obj, **kwargs) def list_container_objects(self, container, prefix=None, delimiter=None): return self.client.get_container(container, prefix, delimiter)[1] def update_object(self, container, obj, metadata, **kwargs): # Format metedata key metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.post_object(container, obj, metadata, **kwargs) def copy_object(self, container, obj, metadata=None, destination=None, **kwargs): return self.client.copy_object(container, obj, headers=metadata, destination=destination, **kwargs)
class ObjectStore: RESP_LIMIT = 10000 # serverside limit of the response def __init__(self, container, name): if name == 'bag_brk': OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18' OBJECTSTORE['tenant_name'] = 'BGE000081_BAG' else: OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '1776010a62684386a08b094d89ce08d9' OBJECTSTORE['tenant_name'] = 'BGE000081_BGT' self.conn = Connection(**OBJECTSTORE) self.container = container def get_store_object(self, name): """ Returns the object store :param name: :return: """ return self.conn.get_object(self.container, name)[1] def get_store_objects(self, path): return self._get_full_container_list([], prefix=path) def _get_full_container_list(self, seed, **kwargs): kwargs['limit'] = self.RESP_LIMIT if len(seed): kwargs['marker'] = seed[-1]['name'] _, page = self.conn.get_container(self.container, **kwargs) seed.extend(page) return seed if len(page) < self.RESP_LIMIT else \ self._get_full_container_list(seed, **kwargs) def folders(self, path): objects_from_store = self._get_full_container_list( [], delimiter='/', prefix=path) return [ store_object['subdir'] for store_object in objects_from_store if 'subdir' in store_object] def files(self, path, file_id): file_list = self._get_full_container_list( [], delimiter='/', prefix=path + file_id) for file_object in file_list: file_object['container'] = self.container return file_list def put_to_objectstore(self, object_name, object_content, content_type): return self.conn.put_object(self.container, object_name, contents=object_content, content_type=content_type) def delete_from_objectstore(self, object_name): return self.conn.delete_object(self.container, object_name)
class Swift(object): def __init__(self, bucket, noop): """Setup the S3 storage backend with the bucket we will use and optional region.""" # This is our Swift container self.bucket = bucket self.noop = noop # We assume your environment variables are set correctly just like # you would for the swift command line util try: self.conn = Connection(authurl=os.environ["ST_AUTH"], user=os.environ["ST_USER"], key=os.environ["ST_KEY"]) except KeyError: logger.warning("Missing environment variables for Swift authentication") logger.warning("Bailing...") sys.exit(1) headers, objs = self.conn.get_account(self.bucket) for i in objs: logger.debug("Searching for bucket %s == %s" % (self.bucket, i)) if not noop and self.bucket not in objs: self.conn.put_container(self.bucket) def list(self, prefix=None): """Return all keys in this bucket.""" headers, objs = self.conn.get_container(self.bucket, prefix=prefix) while objs: # Handle paging i = {} for i in objs: yield i["name"] headers, objs = self.conn.get_container(self.bucket, marker=i["name"], prefix=prefix) def get(self, src): """Return the contents of src from S3 as a string.""" try: headers, obj = self.conn.get_object(self.bucket, src) return obj except ClientException: # Request failed....object doesn't exist return None def put(self, dst, data): """Store the contents of the string data at a key named by dst in S3.""" if self.noop: logger.info("No-Op Put: %s" % dst) else: self.conn.put_object(self.bucket, dst, data) def delete(self, src): """Delete the object in S3 referenced by the key name src.""" if self.noop: logger.info("No-Op Delete: %s" % src) else: self.conn.delete_object(self.bucket, src)
class ObjectStore: RESP_LIMIT = 10000 # serverside limit of the response def __init__(self, container, name): if name == 'bag_brk': OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18' OBJECTSTORE['tenant_name'] = 'BGE000081_BAG' else: OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '1776010a62684386a08b094d89ce08d9' OBJECTSTORE['tenant_name'] = 'BGE000081_BGT' self.conn = Connection(**OBJECTSTORE) self.container = container def get_store_object(self, name): """ Returns the object store :param name: :return: """ return self.conn.get_object(self.container, name)[1] def get_store_objects(self, path): return self._get_full_container_list([], prefix=path) def _get_full_container_list(self, seed, **kwargs): kwargs['limit'] = self.RESP_LIMIT if len(seed): kwargs['marker'] = seed[-1]['name'] _, page = self.conn.get_container(self.container, **kwargs) seed.extend(page) return seed if len(page) < self.RESP_LIMIT else \ self._get_full_container_list(seed, **kwargs) def folders(self, path): objects_from_store = self._get_full_container_list([], delimiter='/', prefix=path) return [ store_object['subdir'] for store_object in objects_from_store if 'subdir' in store_object ] def files(self, path, file_id): file_list = self._get_full_container_list([], delimiter='/', prefix=path + file_id) for file_object in file_list: file_object['container'] = self.container return file_list def put_to_objectstore(self, object_name, object_content, content_type): return self.conn.put_object(self.container, object_name, contents=object_content, content_type=content_type) def delete_from_objectstore(self, object_name): return self.conn.delete_object(self.container, object_name)