class SwiftStorageEngine(BaseStorageEngine): def __init__(self, authurl, user, key, **kwargs): self.client = Connection(authurl, user, key, **kwargs) def save(self, fp, location): location = location.strip("/") i = location.rfind("/") container = location[:i] object_ = location[i:] try: return self.client.put_object(container, object_, fp.read()) except ClientException as ex: logger.error("Failed to get media object: {0}" % location) raise ex def read(self, location): location = location.strip("/") i = location.rfind("/") container = location[:i] object_ = location[i:] try: return self.client.get_object(container, object_)[1] except ClientException as ex: logger.error("Failed to get media object: {0}" % location) raise ex def get_url(self, request, location): return request.url('media', location=location)
def __init__(self, session=None, swift_storage_url=None, swift_auth_token=None, swift_auth_v1_url=None, swift_user=None, swift_key=None): """Create new OpenStack Swift Object Storage Session. Three methods are possible: 1. Create session by the swiftclient library. 2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs). 3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library). Parameters ---------- session: optional A swiftclient connection object swift_storage_url: the storage URL swift_auth_token: the value of the x-auth-token authorization token swift_storage_url: string, optional authentication URL swift_user: string, optional user name to authenticate as swift_key: string, optional key/password to authenticate with Examples -------- >>> import rasterio >>> from rasterio.session import SwiftSession >>> fp = '/vsiswift/bucket/key.tif' >>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing') >>> session = SwiftSession(conn) >>> with rasterio.Env(session): >>> with rasterio.open(fp) as src: >>> print(src.profile) """ if swift_storage_url and swift_auth_token: self._creds = { "swift_storage_url": swift_storage_url, "swift_auth_token": swift_auth_token } else: from swiftclient.client import Connection if session: self._session = session else: self._session = Connection(authurl=swift_auth_v1_url, user=swift_user, key=swift_key) self._creds = { "swift_storage_url": self._session.get_auth()[0], "swift_auth_token": self._session.get_auth()[1] }
def get_auth(): conn = Connection( authurl=os.environ['OS_AUTH_URL'], user=os.environ['OS_USERNAME'], key=os.environ['OS_PASSWORD'], tenant_name=os.environ['OS_TENANT_NAME'], auth_version=2.0) return conn.get_auth()
def _setup(self): auth = v3.Password(auth_url=self.auth_url, user_domain_name=self.user_domain_name, username=self.username, password=self.password, project_domain_name=self.project_domain_name, project_name=self.project_name) sess = session.Session(auth=auth) self.client = Connection(auth_version=self.auth_version, session=sess) self.quota = OpenStackQuota(self.client, self.tenant_id, self.limit)
class SwiftStorage(object): def __init__(self, account, url, user, key): self.conn = Connection(authurl=url, user=user, key=key) self.cont = self.conn.get_container(account) self.account = account def list(self): return [x["name"] for x in self.cont[1]] def get_attrs(self, uuid): attrs = {} k = self.conn.get_object(container=self.account, obj=uuid)[0] print k attrs['tags'] = json.loads(k['x-object-meta-tags']) attrs['flags'] = json.loads(k['x-object-meta-flags']) attrs['stored'] = int(k['x-object-meta-stored']) return attrs def get_message(self, uuid): return self.conn.get_object(container=self.account, obj=uuid)[1] def put_message(self, uuid, msg, attrs): self.conn.put_object(container=self.account, obj=uuid, contents=msg, headers=attrs) self.put_attrs(uuid, attrs) def put_attrs(self, uuid, attrs): newattrs = {'X-Object-Meta-Tags': json.dumps(attrs['tags']), 'X-Object-Meta-Flags': json.dumps(attrs['flags']), 'X-Object-Meta-Stored': str(attrs['stored'])} print newattrs self.conn.post_object(container=self.account, obj=uuid, headers=newattrs) def del_message(self, uuid): self.conn.delete_object(container=self.account, obj=uuid)
def add_client(self): conn = Connection(authurl=self.config.auth_url, user=self.config.user, key=self.config.key, retries=self.config.retries, snet=self.config.is_snet, starting_backoff=float(self.config.starting_backoff), max_backoff=float(self.config.max_backoff), tenant_name=self.config.tenant_name, os_options=parse_extra_into_dict(self.config.custom_options), auth_version=self.config.auth_version, cacert=self.config.cacert, insecure=not self.config.should_validate_cert, ssl_compression=self.config.needs_tls_compr, retry_on_ratelimit=self.config.should_retr_ratelimit) try: conn.head_account() except Exception, e: self.logger.warn('Could not HEAD an account (%s), e:`%s`', self.config.name, format_exc(e))
def create_swift_client(self): try: swift = SwiftClient(auth_version='3', os_options=self.__keystoneCredentials, authurl=self.__keystoneCredentials['auth_url'], user=self.__keystoneCredentials['username'], key=self.__keystoneCredentials['password'], cacert=self.__certificatesPath) swift.get_auth() return swift except Exception as error: raise Exception('Connection to Swift failed: {}'.format(error))
def add_client(self): conn = Connection(authurl=self.config.auth_url, user=self.config.user, key=self.config.key, retries=self.config.retries, snet=self.config.is_snet, starting_backoff=float(self.config.starting_backoff), max_backoff=float(self.config.max_backoff), tenant_name=self.config.tenant_name, os_options=parse_extra_into_dict(self.config.custom_options), auth_version=self.config.auth_version, cacert=self.config.cacert, insecure=not self.config.should_validate_cert, ssl_compression=self.config.needs_tls_compr, retry_on_ratelimit=self.config.should_retr_ratelimit) try: conn.head_account() except Exception: self.logger.warn('Could not HEAD an account (%s), e:`%s`', self.config.name, format_exc()) self.client.put_client(conn)
class Main(object): def __init__(self): parser = OptionParser( usage="%prog [options] memstore_admin password", version="%prog v" + VERSION, description="List Miniserver snapshots stored in a Memstore(tm) instance.", epilog="For further information, please visit: http://www.memset.com/cloud/storage/", ) parser.add_option( "--auth_url", type="str", dest="auth_url", default=AUTH_URL, help="Auth URL (default: %s)" % AUTH_URL ) self.options, self.args = parser.parse_args() if len(self.args) != 2: parser.error("Not enough parameters provided") self.conn = Connection(self.options.auth_url, self.args[0], self.args[1]) def run(self): info, snapshots = self.conn.get_container("miniserver-snapshots", delimiter="/") for snap in snapshots: if "subdir" not in snap.keys(): continue _, objects = self.conn.get_container("miniserver-snapshots", prefix=snap["subdir"]) snap["bytes-used"] = 0 snap["parts"] = 0 readme = False for obj in objects: snap["bytes-used"] += obj["bytes"] if ".part" in obj.get("name", ""): snap["parts"] += 1 elif obj.get("name", "").lower().endswith("readme.txt"): readme = True snap["date"] = obj["last_modified"] if not readme: snap["notes"] = "README.txt not found, it may be an incomplete snapshot" result = dict( (key[len("x-container-") :], value) for key, value in info.items() if key.startswith("x-container-") ) result["snapshots"] = snapshots print json.dumps(result, indent=1)
def get_swift_connection(): # find api version for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): if k in os.environ: auth_version = os.environ[k] break else: auth_version = 1 # cast from string to int auth_version = int(float(auth_version)) if auth_version == 3: # keystone v3 try: auth_url = os.environ['OS_AUTH_URL'] auth_user = os.environ['OS_USERNAME'] auth_password = os.environ['OS_PASSWORD'] project_name = os.environ['OS_PROJECT_NAME'] except KeyError: raise UsageError( "You need to set OS_AUTH_URL, OS_USERNAME, OS_PASSWORD and " "OS_PROJECT_NAME for Swift authentication") auth_os_options = { 'user_domain_name': os.environ.get('OS_USER_DOMAIN_NAME', 'Default'), 'project_domain_name': os.environ.get('OS_PROJECT_DOMAIN_NAME', 'Default'), 'project_name': project_name } return Connection(auth_url, auth_user, auth_password, os_options=auth_os_options, auth_version='3') elif auth_version == 2: # keystone v2 (not implemented) raise NotImplementedError('keystone v2 is not supported') else: try: auth_url = os.environ['ST_AUTH'] auth_user = os.environ['ST_USER'] auth_password = os.environ['ST_KEY'] except KeyError: raise UsageError("You need to set ST_AUTH, ST_USER, ST_KEY for " "Swift authentication") return Connection(auth_url, auth_user, auth_password)
def prepareffmpeg(url): _authurl = "http://" + url + ":8080/auth/v1.0" conn = Connection(authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version) _, ffmpegExec = conn.get_object(container_name, "ffmpeg") with open(FFMPEG_PATH, 'w') as local: local.write(ffmpegExec) #shutil.copyfile('ffmpeg', FFMPEG_PATH) os.chmod(FFMPEG_PATH, 0o0755)
def main(args): output_bucket = args.get("output_bucket") mapper_bucket = args["mapper_bucket"] input_bucket = args["input_bucket"] n_mapper = args["n_mapper"] _authurl = "http://"+args.get('url')+":8080/auth/v1.0" conn = Connection( authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version ) # Fetch all the keys all_keys = [] for obj in conn.get_container(input_bucket)[1]: all_keys.append(obj['name']) total_size = len(all_keys) batch_size = 0 if total_size % n_mapper == 0: batch_size = total_size / n_mapper else: batch_size = total_size // n_mapper + 1 for idx in range(n_mapper): print("mapper-" + str(idx) + ":" + str(all_keys[idx * batch_size: (idx + 1) * batch_size])) pool = ThreadPool(n_mapper) invoke_mapper_partial = partial(invoke_mapper_action, "default/mapper", mapper_bucket, input_bucket, all_keys, batch_size) pool.map(invoke_mapper_partial, range(n_mapper)) pool.close() pool.join() while True: res_s3 = conn.get_container(mapper_bucket)[1] if "Contents" not in res_s3.keys(): job_keys = [] else: job_keys = res_s3["Contents"] print("Wait Mapper Jobs ...") time.sleep(5) if len(job_keys) == n_mapper: print("[*] Map Done : mapper " + str(len(job_keys)) + " finished.") break return {"input_reducer_bucket": mapper_bucket, "output_bucket": output_bucket}
def __init__(self, container, name): if name == 'bag_brk': OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18' OBJECTSTORE['tenant_name'] = 'BGE000081_BAG' else: OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '1776010a62684386a08b094d89ce08d9' OBJECTSTORE['tenant_name'] = 'BGE000081_BGT' self.conn = Connection(**OBJECTSTORE) self.container = container
def __init__(self, bucket, noop): """Setup the S3 storage backend with the bucket we will use and optional region.""" # This is our Swift container self.bucket = bucket self.noop = noop # We assume your environment variables are set correctly just like # you would for the swift command line util try: self.conn = Connection(authurl=os.environ["ST_AUTH"], user=os.environ["ST_USER"], key=os.environ["ST_KEY"], timeout=30) except KeyError: logger.warning("Missing environment variables for Swift authentication") logger.warning("Bailing...") sys.exit(1) headers, objs = self.conn.get_account(self.bucket) for i in objs: logger.debug("Searching for bucket %s == %s" % (self.bucket, i)) if not noop and self.bucket not in objs: self.conn.put_container(self.bucket)
def __init__(self, container, name): if name == 'gob': OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = GOB_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '2ede4a78773e453db73f52500ef748e5' OBJECTSTORE['tenant_name'] = 'BGE000081 GOB' else: OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '1776010a62684386a08b094d89ce08d9' OBJECTSTORE['tenant_name'] = 'BGE000081_BGT' self.conn = Connection(**OBJECTSTORE) self.container = container
def get_connection(store_settings: dict) -> Connection: """ get an objectsctore connection """ store = store_settings os_options = { 'tenant_id': store['TENANT_ID'], 'region_name': store['REGION_NAME'], # 'endpoint_type': 'internalURL' } # when we are running in cloudvps we should use internal urls use_internal = os.getenv('OBJECTSTORE_LOCAL', '') if use_internal: os_options['endpoint_type'] = 'internalURL' connection = Connection(authurl=store['AUTHURL'], user=store['USER'], key=store['PASSWORD'], tenant_name=store['TENANT_NAME'], auth_version=store['VERSION'], os_options=os_options) return connection
def __init_swift_fileserver(self): try: options = { 'authurl': self.authurl, 'user': self.user, 'key': self.key, 'auth_version': self.auth_version, 'tenant_name': self.tenant_name, 'insecure': True, 'timeout': CONN_TIMEOUT, 'os_options': {'region_name': self.region_name}} headers = {'Temp-URL-Key': self.temp_url_key, 'Temp-URL-Key-2': self.temp_url_key_2 } print_opts = copy.deepcopy(options) print_opts['key'] = '<password stripped>' self.swift_conn = Connection(**options) self.swift_conn.post_account(headers) self.swift_conn.put_container(self.container_name) self.storageurl = self.swift_conn.get_auth()[0] print ('swift-file-server: Connected. options %s storageurl %s', print_opts, self.storageurl) except Exception as err: print('swift-file-server: Connect FAILED %s options %s', err, print_opts) raise
def swift_connect(): conn = Connection(authurl=default_authurl, user=default_user, key=default_key, auth_version=default_auth_version, os_options=None) return conn
def test_load_handled_files_invalid(self, _): store = StoreObjectStorage('handled_files_test', container_name='foobar', connection=Connection()) store.whitelisted.append('foobar') items = store.load_handled_files() self.assertEqual(items, set())
def _auth(self): cnt = Connection(authurl=Configuration.auth_url, user=Configuration.username, key=Configuration.password, tenant_name=Configuration.tenant_name, auth_version=2, insecure=False) return cnt
def upload_output_to_s3(bucketName, filePrefix, url): print 'Uploading files to s3: %s' % bucketName _authurl = "http://" + url + ":8080/auth/v1.0" conn = Connection(authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version) #Perfs counters # s3 = boto3.client('s3', config=botocore.client.Config( # max_pool_connections=MAX_PARALLEL_UPLOADS)) count = 0 totalSize = 0 results = [] pool = ThreadPool(MAX_PARALLEL_UPLOADS) sema = Semaphore(MAX_PARALLEL_UPLOADS) def upload_file(localFilePath, uploadFileName, fileSize): sema.acquire() try: print 'Start: %s [%dKB]' % (localFilePath, fileSize >> 10) with open(localFilePath, 'rb') as ifs: #s3.put_object(Body=ifs, Bucket=bucketName, # Key=uploadFileName, # StorageClass='REDUCED_REDUNDANCY') conn.put_object(bucketName, uploadFileName, contents=ifs) #, content_type="image/jpeg") print 'Done: %s' % localFilePath finally: sema.release() for fileName in list_output_files(): localFilePath = os.path.join(TEMP_OUTPUT_DIR, fileName) uploadFileName = os.path.join(filePrefix, fileName) fileSize = os.path.getsize(localFilePath) result = pool.apply_async(upload_file, args=(localFilePath, uploadFileName, fileSize)) results.append(result) count += 1 totalSize += fileSize # block until all threads are done for result in results: result.get() # block until all uploads are finished for _ in xrange(MAX_PARALLEL_UPLOADS): sema.acquire() print 'Uploaded %d files to Swift [total=%dKB]' % (count, totalSize >> 10) return (count, totalSize)
def __init__(self, session=None, swift_storage_url=None, swift_auth_token=None, swift_auth_v1_url=None, swift_user=None, swift_key=None): """Create new OpenStack Swift Object Storage Session. Three methods are possible: 1. Create session by the swiftclient library. 2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs). 3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library). Parameters ---------- session: optional A swiftclient connection object swift_storage_url: the storage URL swift_auth_token: the value of the x-auth-token authorization token swift_storage_url: string, optional authentication URL swift_user: string, optional user name to authenticate as swift_key: string, optional key/password to authenticate with Examples -------- >>> import rasterio >>> from rasterio.session import SwiftSession >>> fp = '/vsiswift/bucket/key.tif' >>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing') >>> session = SwiftSession(conn) >>> with rasterio.Env(session): >>> with rasterio.open(fp) as src: >>> print(src.profile) """ if swift_storage_url and swift_auth_token: self._creds = { "swift_storage_url": swift_storage_url, "swift_auth_token": swift_auth_token } else: from swiftclient.client import Connection if session: self._session = session else: self._session = Connection( authurl=swift_auth_v1_url, user=swift_user, key=swift_key ) self._creds = { "swift_storage_url": self._session.get_auth()[0], "swift_auth_token": self._session.get_auth()[1] }
def _get_client(self): auth = v3.Password( auth_url=self.auth_url, username=os.getenv('INFRABOX_STORAGE_SWIFT_USERNAME'), password=os.getenv('INFRABOX_STORAGE_SWIFT_PASSWORD'), user_domain_name=self.user_domain_name, project_name=self.project_name, project_domain_name=self.project_domain_name) keystone_session = session.Session(auth=auth) return Connection(session=keystone_session)
def _setup(self): auth = v3.Password(auth_url=self.auth_url, user_domain_name=self.user_domain_name, username=self.username, password=self.password, project_domain_name=self.project_domain_name, project_name=self.project_name) sess = session.Session(auth=auth) self.client = Connection(auth_version=self.auth_version, session=sess) self.quota = OpenStackQuota( self.client, self.tenant_id, self.limit)
def _get_connection(self): return Connection( authurl=self._auth_url, cacert=self._ca_cert_path, user=self._swift_user, key=self._swift_password, auth_version=self._auth_version, os_options=self._os_options, retry_on_ratelimit=self._retry_on_ratelimit, timeout=self._connect_timeout or _DEFAULT_SWIFT_CONNECT_TIMEOUT, retries=self._retry_count or 5, )
def get_connection_v1(user, key): """ conn : swiftclient connection """ _authurl = "http://127.0.0.1:8080/auth/v1.0" _auth_version = '1' _user = user _key = key conn = Connection(authurl=_authurl, user=_user, key=_key) return conn
def swift_conn(authurl, user, project, password): ''' authurl : http://keystone:5000 user: aaa project: aaa password: bbb ''' # by default add http schema if not authurl.startswith("http"): authurl = "http://" + authurl os_options = {"tenant_name": project} conn = SwiftConnection(authurl=authurl, user=user, key=password, os_options=os_options, auth_version=3) conn.get_auth() yield conn conn.close()
def main(args): output_bucket = args.get("output_bucket") input_reducer_bucket = args.get("input_reducer_bucket") _authurl = "http://" + args.get('url') + ":8080/auth/v1.0" conn = Connection(authurl=_authurl, user=_user, key=_key, tenant_name=_tenant_name, auth_version=_auth_version) all_keys = [] for obj in conn.get_container(input_reducer_bucket)[1]: all_keys.append(obj['name']) final_str = [] for key in all_keys: start = time() response = conn.get_object(input_reducer_bucket, key) contents = response.read() contents = contents.decode() final_str.append(to_string(getResultReduce(contents))) final_str = "\n".join(final_str) print(final_str) final_res = getResultReduce(final_str) conn.put_object(output_bucket, "reduce_res.json", contents=json.dumps(final_res)) return {"res": "good"}
def run(self, terms, variables=None, **kwargs): """Returns Ironic Inspector introspection data. Access swift and return introspection data for all nodes. :returns a list of tuples, one for each node. """ ret = [] session = get_auth_session(kwargs.get('auth_url'), "ironic", "service", kwargs.get('password')) swift_client = Connection(session=session) container = swift_client.get_container("ironic-inspector") for item in container[1]: if item['name'].startswith('inspector_data') and \ not item['name'].endswith("UNPROCESSED"): obj = swift_client.get_object("ironic-inspector", item['name']) ret.append((item['name'], obj)) return ret
def get_swift_connection_userdocs(): """ Function which returns a connection to a swift object storage :return connection: connection to swift :rtype connection: swiftclient.client.Connection """ swift = Connection(authurl=settings.OS_AUTH_URL, user=settings.OS_USERNAME, key=settings.OS_PASSWORD, tenant_name=settings.OS_TENANT_NAME, auth_version=settings.OS_IDENTITY_API_VERSION, os_options=settings.OS_OPTIONS) return swift
def upload_file(container, input_file, output_file): auth = v3.Password(auth_url = swift_auth, username = swift_me, password = swift_pass, user_domain_name='Default', project_name='Data Science', project_domain_name='Default') # Create session keystone_session = session.Session(auth=auth) # Create swiftclient Connection swift_conn = Connection(session=keystone_session) with open(input_file, 'rb') as local: swift_conn.put_object( container, output_file, contents=local, content_type='text/plain', chunk_size = 1024*100 )
def __init__(self, bucket, noop): """Setup the S3 storage backend with the bucket we will use.""" # This is our Swift container self.bucket = bucket self.noop = noop # We assume your environment variables are set correctly just like # you would for the swift command line util try: self.conn = Connection(authurl=os.environ["ST_AUTH"], user=os.environ["ST_USER"], key=os.environ["ST_KEY"]) except KeyError: logger.warning( "Missing environment variables for Swift authentication") logger.warning("Bailing...") sys.exit(1) headers, objs = self.conn.get_account(self.bucket) for i in objs: logger.debug("Searching for bucket %s == %s" % (self.bucket, i)) if not noop and self.bucket not in objs: self.conn.put_container(self.bucket)
def swift_connect(_authurl=None, _auth_version=None, _user=None, _key=None): if (_authurl == None): _authurl = default_authurl if (_auth_version == None): _auth_version = default_auth_version if (_user == None): _user = default_user if (_key == None): _key = default_key conn = Connection(authurl=_authurl, user=_user, key=_key, auth_version=_auth_version, os_options=default_os_options) return conn
def swift_client(context): if CONF.swift_url: # swift_url has a different format so doesn't need to be normalized url = '%(swift_url)s%(tenant)s' % {'swift_url': CONF.swift_url, 'tenant': context.tenant} else: url = get_endpoint(context.service_catalog, service_type=CONF.swift_service_type, endpoint_region=CONF.os_region_name, endpoint_type=CONF.swift_endpoint_type) client = Connection(preauthurl=url, preauthtoken=context.auth_token, tenant_name=context.tenant, snet=USE_SNET) return client
def createConnection(config): # Create a password auth plugin auth = v3.Password( auth_url=config.p.conf_file.get_swift_auth_url(), username=config.p.conf_file.get_swift_user(), password=config.p.conf_file.get_swift_password(), user_domain_name=config.p.conf_file.get_swift_user_domain_name(), project_name=config.p.conf_file.get_swift_project_name(), project_id=config.p.conf_file.get_swift_project_id()) # Create session sessionKeystone = session.Session(auth=auth, verify=False) # Create swiftclient Connection global swift swift = Connection(insecure=True, session=sessionKeystone)
def __init__(self): parser = OptionParser( usage="%prog [options] memstore_admin password", version="%prog v" + VERSION, description="List Miniserver snapshots stored in a Memstore(tm) instance.", epilog="For further information, please visit: http://www.memset.com/cloud/storage/", ) parser.add_option( "--auth_url", type="str", dest="auth_url", default=AUTH_URL, help="Auth URL (default: %s)" % AUTH_URL ) self.options, self.args = parser.parse_args() if len(self.args) != 2: parser.error("Not enough parameters provided") self.conn = Connection(self.options.auth_url, self.args[0], self.args[1])
def swift_client(context, region_name=None): if CONF.swift_url: # swift_url has a different format so doesn't need to be normalized url = '%(swift_url)s%(tenant)s' % { 'swift_url': CONF.swift_url, 'tenant': context.project_id } else: region = region_name or CONF.service_credentials.region_name url = get_endpoint(context.service_catalog, service_type=CONF.swift_service_type, endpoint_region=region, endpoint_type=CONF.swift_endpoint_type) client = Connection(preauthurl=url, preauthtoken=context.auth_token, tenant_name=context.project_id, snet=CONF.backup_use_snet, insecure=CONF.swift_api_insecure) return client
def get_connection(): options = { 'tenant_id': store['TENANT_ID'], 'region_name': store['REGION_NAME'], 'endpoint_type': 'internalURL', } log.debug("Do we run local? set ENV LOCAL to something") if os.getenv('LOCAL', False): log.debug("We run LOCAL!") options.pop('endpoint_type') new_handelsregister_conn = Connection(authurl=store['AUTHURL'], user=store['USER'], key=store['PASSWORD'], tenant_name=store['TENANT_NAME'], auth_version=store['VERSION'], os_options=options) return new_handelsregister_conn
def objectstore_connection(config_full_path, config_name, print_config_vars=None): """ Get an objectsctore connection. Args: 1. config_full_path: /path_to_config/config.ini or config.ini if in root. 2. config_name: objectstore 3. print_config_vars: if set to True: print all variables from the config file Returns: An objectstore connection session. """ assert os.environ['OBJECTSTORE_PASSWORD'] config = get_config(config_full_path) if print_config_vars: logger.info('config variables.. :{}'.format(OBJECTSTORE)) conn = Connection( authurl=config.get(config_name, 'AUTHURL'), user=config.get(config_name, 'USER'), key=os.environ['OBJECTSTORE_PASSWORD'], tenant_name=config.get(config_name, 'TENANT_NAME'), auth_version=config.get(config_name, 'VERSION'), os_options={ 'tenant_id': config.get(config_name, 'TENANT_ID'), 'region_name': config.get(config_name, 'REGION_NAME'), # 'endpoint_type': 'internalURL' }) logger.info('Established successfull connection to {}'.format( config.get(config_name, 'TENANT_NAME'))) return conn
def __init__(self, authurl, user, key, **kwargs): self.client = Connection(authurl, user, key, **kwargs)
def __init__(self, account, url, user, key): self.conn = Connection(authurl=url, user=user, key=key) self.cont = self.conn.get_container(account) self.account = account
class SwiftSession(Session): """Configures access to secured resources stored in OpenStack Swift Object Storage. """ def __init__(self, session=None, swift_storage_url=None, swift_auth_token=None, swift_auth_v1_url=None, swift_user=None, swift_key=None): """Create new OpenStack Swift Object Storage Session. Three methods are possible: 1. Create session by the swiftclient library. 2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs). 3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library). Parameters ---------- session: optional A swiftclient connection object swift_storage_url: the storage URL swift_auth_token: the value of the x-auth-token authorization token swift_storage_url: string, optional authentication URL swift_user: string, optional user name to authenticate as swift_key: string, optional key/password to authenticate with Examples -------- >>> import rasterio >>> from rasterio.session import SwiftSession >>> fp = '/vsiswift/bucket/key.tif' >>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing') >>> session = SwiftSession(conn) >>> with rasterio.Env(session): >>> with rasterio.open(fp) as src: >>> print(src.profile) """ if swift_storage_url and swift_auth_token: self._creds = { "swift_storage_url": swift_storage_url, "swift_auth_token": swift_auth_token } else: from swiftclient.client import Connection if session: self._session = session else: self._session = Connection(authurl=swift_auth_v1_url, user=swift_user, key=swift_key) self._creds = { "swift_storage_url": self._session.get_auth()[0], "swift_auth_token": self._session.get_auth()[1] } @classmethod def hascreds(cls, config): """Determine if the given configuration has proper credentials Parameters ---------- cls : class A Session class. config : dict GDAL configuration as a dict. Returns ------- bool """ return 'SWIFT_STORAGE_URL' in config and 'SWIFT_AUTH_TOKEN' in config @property def credentials(self): """The session credentials as a dict""" return self._creds def get_credential_options(self): """Get credentials as GDAL configuration options Returns ------- dict """ return {k.upper(): v for k, v in self.credentials.items()}
class SwiftSession(Session): """Configures access to secured resources stored in OpenStack Swift Object Storage. """ def __init__(self, session=None, swift_storage_url=None, swift_auth_token=None, swift_auth_v1_url=None, swift_user=None, swift_key=None): """Create new OpenStack Swift Object Storage Session. Three methods are possible: 1. Create session by the swiftclient library. 2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs). 3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library). Parameters ---------- session: optional A swiftclient connection object swift_storage_url: the storage URL swift_auth_token: the value of the x-auth-token authorization token swift_storage_url: string, optional authentication URL swift_user: string, optional user name to authenticate as swift_key: string, optional key/password to authenticate with Examples -------- >>> import rasterio >>> from rasterio.session import SwiftSession >>> fp = '/vsiswift/bucket/key.tif' >>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='******', key='testing') >>> session = SwiftSession(conn) >>> with rasterio.Env(session): >>> with rasterio.open(fp) as src: >>> print(src.profile) """ if swift_storage_url and swift_auth_token: self._creds = { "swift_storage_url": swift_storage_url, "swift_auth_token": swift_auth_token } else: from swiftclient.client import Connection if session: self._session = session else: self._session = Connection( authurl=swift_auth_v1_url, user=swift_user, key=swift_key ) self._creds = { "swift_storage_url": self._session.get_auth()[0], "swift_auth_token": self._session.get_auth()[1] } @classmethod def hascreds(cls, config): """Determine if the given configuration has proper credentials Parameters ---------- cls : class A Session class. config : dict GDAL configuration as a dict. Returns ------- bool """ return 'SWIFT_STORAGE_URL' in config and 'SWIFT_AUTH_TOKEN' in config @property def credentials(self): """The session credentials as a dict""" return self._creds def get_credential_options(self): """Get credentials as GDAL configuration options Returns ------- dict """ return {k.upper(): v for k, v in self.credentials.items()}
class Swift(object): def __init__(self, bucket, noop): """Setup the S3 storage backend with the bucket we will use and optional region.""" # This is our Swift container self.bucket = bucket self.noop = noop # We assume your environment variables are set correctly just like # you would for the swift command line util try: self.conn = Connection(authurl=os.environ["ST_AUTH"], user=os.environ["ST_USER"], key=os.environ["ST_KEY"]) except KeyError: logger.warning("Missing environment variables for Swift authentication") logger.warning("Bailing...") sys.exit(1) headers, objs = self.conn.get_account(self.bucket) for i in objs: logger.debug("Searching for bucket %s == %s" % (self.bucket, i)) if not noop and self.bucket not in objs: self.conn.put_container(self.bucket) def list(self, prefix=None): """Return all keys in this bucket.""" headers, objs = self.conn.get_container(self.bucket, prefix=prefix) while objs: # Handle paging i = {} for i in objs: yield i["name"] headers, objs = self.conn.get_container(self.bucket, marker=i["name"], prefix=prefix) def get(self, src): """Return the contents of src from S3 as a string.""" try: headers, obj = self.conn.get_object(self.bucket, src) return obj except ClientException: # Request failed....object doesn't exist return None def put(self, dst, data): """Store the contents of the string data at a key named by dst in S3.""" if self.noop: logger.info("No-Op Put: %s" % dst) else: self.conn.put_object(self.bucket, dst, data) def delete(self, src): """Delete the object in S3 referenced by the key name src.""" if self.noop: logger.info("No-Op Delete: %s" % src) else: self.conn.delete_object(self.bucket, src)
class ObjectStore: RESP_LIMIT = 10000 # serverside limit of the response def __init__(self, container, name): if name == 'bag_brk': OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BAG_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '4f2f4b6342444c84b3580584587cfd18' OBJECTSTORE['tenant_name'] = 'BGE000081_BAG' else: OBJECTSTORE['user'] = '******' OBJECTSTORE['key'] = BGT_OBJECTSTORE_PASSWORD OBJECTSTORE['os_options'][ 'tenant_id'] = '1776010a62684386a08b094d89ce08d9' OBJECTSTORE['tenant_name'] = 'BGE000081_BGT' self.conn = Connection(**OBJECTSTORE) self.container = container def get_store_object(self, name): """ Returns the object store :param name: :return: """ return self.conn.get_object(self.container, name)[1] def get_store_objects(self, path): return self._get_full_container_list([], prefix=path) def _get_full_container_list(self, seed, **kwargs): kwargs['limit'] = self.RESP_LIMIT if len(seed): kwargs['marker'] = seed[-1]['name'] _, page = self.conn.get_container(self.container, **kwargs) seed.extend(page) return seed if len(page) < self.RESP_LIMIT else \ self._get_full_container_list(seed, **kwargs) def folders(self, path): objects_from_store = self._get_full_container_list( [], delimiter='/', prefix=path) return [ store_object['subdir'] for store_object in objects_from_store if 'subdir' in store_object] def files(self, path, file_id): file_list = self._get_full_container_list( [], delimiter='/', prefix=path + file_id) for file_object in file_list: file_object['container'] = self.container return file_list def put_to_objectstore(self, object_name, object_content, content_type): return self.conn.put_object(self.container, object_name, contents=object_content, content_type=content_type) def delete_from_objectstore(self, object_name): return self.conn.delete_object(self.container, object_name)
class SwiftFileServer(object): """ Swift File server """ def __init__(self, region_name): # self.cfg = cfg.CONF self.region_name = region_name # self.authurl = self.cfg.FileSvc.authurl # self.auth_version = self.cfg.FileSvc.auth_version # self.user = self.cfg.FileSvc.user # self.key = self.cfg.FileSvc.key # self.tenant_name = self.cfg.FileSvc.tenant_name # self.container_name = self.cfg.FileSvc.container_name # self.temp_url_key = self.cfg.FileSvc.temp_url_key # self.temp_url_key_2 = self.cfg.FileSvc.temp_url_key_2 # self.chosen_temp_url_key = self.cfg.FileSvc.chosen_temp_url_key # self.authurl='http://10.204.248.50:35357/v2.0' self.authurl = 'http://10.204.248.228:35357/v2.0' self.auth_version='2.0' self.user='******' self.key='passw0rd' self.tenant_name='admin' self.temp_url_key= 'mykey' self.temp_url_key_2='mykey2' self.chosen_temp_url_key= 'temp_url_key' self.container_name= 'mycontainer' self.storageurl = None self.swift_conn = None def connect_to_swift(self): """ return connect to swift fileserver """ for i in range(RETRY_CNT): i += 1 try: self.__init_swift_fileserver() break except Exception as err: print('[Try %d/%d ]: Connecting swift fileserver failed. %s %s', i, RETRY_CNT, err, traceback.format_exc()) if i == RETRY_CNT: raise def __init_swift_fileserver(self): try: options = { 'authurl': self.authurl, 'user': self.user, 'key': self.key, 'auth_version': self.auth_version, 'tenant_name': self.tenant_name, 'insecure': True, 'timeout': CONN_TIMEOUT, 'os_options': {'region_name': self.region_name}} headers = {'Temp-URL-Key': self.temp_url_key, 'Temp-URL-Key-2': self.temp_url_key_2 } print_opts = copy.deepcopy(options) print_opts['key'] = '<password stripped>' self.swift_conn = Connection(**options) self.swift_conn.post_account(headers) self.swift_conn.put_container(self.container_name) self.storageurl = self.swift_conn.get_auth()[0] print ('swift-file-server: Connected. options %s storageurl %s', print_opts, self.storageurl) except Exception as err: print('swift-file-server: Connect FAILED %s options %s', err, print_opts) raise def upload_file(self, fpath, fname=None,expires=3600): """ upload the file in 'filepath' on to the file server and return a temporary url for the users to download """ if not self.swift_conn: self.connect_to_swift() if fname is None: fname = os.path.basename(fpath) finp = open(fpath, 'rb') try: self.swift_conn.put_object( container=self.container_name, obj=fname, contents=finp) print ('swift-file-server: Uploading file %s ... [OK]', fname) except Exception as err: logging.error( 'swift-file-server: Unable to upload the file %s: %s', fname, err) raise # return self.get_temp_download_url(fname,expires) def get_temp_download_url(self, fname, expires): """ return the temporary download url """ file_uri = '%s/%s/%s' % (self.storageurl, self.container_name, fname) file_path = urlparse(file_uri).path key = getattr(self, self.chosen_temp_url_key) try: temp_url = generate_temp_url(file_path, expires, key, 'GET') except Exception as err: logging.error( 'swift-file-server: Generating temp url for %s failed %s', fname, err) raise download_url = self.storageurl.replace( urlparse(self.storageurl).path, temp_url) print( 'swift-file-server: Temporary download URL for file %s: %s', fname, download_url) return download_url def delete_file(self, fpath): """ Delete the file from the file server """ fname = os.path.basename(fpath) try: self.swift_conn.delete_object( container=self.container_name, obj=fname) print('swift-file-server: Deleting file %s ... [OK]', fname) except Exception as err: print( 'swift-file-server: Deleting file %s ... [FAIL]: %s %s (IGNORED) ', fname, err, traceback.format_exc()) def getObjectContent(self, filename): return self.swift_conn.get_object(container=self.container_name, obj=filename)
class OpenstackDriver(BaseDriver): """OpenStackDriver for Object Storage""" def __init__(self, cloud_config): super(OpenstackDriver, self).__init__() self.auth_url = cloud_config['os_auth_url'] self.project_name = cloud_config['os_project_name'] self.username = cloud_config['os_username'] self.password = cloud_config['os_password'] self.user_domain_name = \ cloud_config.get('os_project_domain_name', 'default') self.project_domain_name = \ cloud_config.get('os_user_domain_name', 'default') self.driver_name = \ cloud_config.get('driver_name', 'default') self.tenant_id = cloud_config.get('tenant_id', None) self.limit = cloud_config.get('limit', None) self.auth_version = \ cloud_config.get('os_auth_version', '2') self._setup() def _setup(self): auth = v3.Password(auth_url=self.auth_url, user_domain_name=self.user_domain_name, username=self.username, password=self.password, project_domain_name=self.project_domain_name, project_name=self.project_name) sess = session.Session(auth=auth) self.client = Connection(auth_version=self.auth_version, session=sess) self.quota = OpenStackQuota( self.client, self.tenant_id, self.limit) def create_container(self, container, **kwargs): return self.client.put_container(container) def delete_container(self, container): return self.client.delete_container(container) def list_containers(self): return self.client.get_account()[1] def stat_container(self, container): return self.client.head_container(container) def update_container(self, container, metadata, **kwargs): metadata = {('x-container-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-container-meta-')} return self.client.post_container(container, metadata, **kwargs) def upload_object(self, container, obj, contents, content_length=None, metadata=None, **kwargs): if metadata: metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.put_object(container, obj, contents=contents, content_length=content_length, headers=metadata, **kwargs) def download_object(self, container, obj, **kwargs): return self.client.get_object(container, obj, **kwargs) def stat_object(self, container, obj): return self.client.head_object(container, obj) def delete_object(self, container, obj, **kwargs): return self.client.delete_object(container, obj, **kwargs) def list_container_objects(self, container, prefix=None, delimiter=None): return self.client.get_container(container, prefix, delimiter)[1] def update_object(self, container, obj, metadata, **kwargs): # Format metedata key metadata = {('x-object-meta-' + key.strip()): value for key, value in metadata.items() if not key.strip().startswith('x-object-meta-')} return self.client.post_object(container, obj, metadata, **kwargs) def copy_object(self, container, obj, metadata=None, destination=None, **kwargs): return self.client.copy_object(container, obj, headers=metadata, destination=destination, **kwargs)