def set_ins_object_from_json(bucket, key, data): """Set the value of 'key' in 'bucket' to equal to contents of 'data', which has been encoded to json, if (and only if) this key has not already been set (ins = 'if not set'). This returns the object at this key after the operation (either the set object or the value that was previously set """ from Acquire.ObjectStore import Mutex as _Mutex m = _Mutex(bucket=bucket, key=key) try: old_data = ObjectStore.get_object_from_json(bucket, key) except: old_data = None if old_data is not None: m.unlock() return old_data else: try: ObjectStore.set_object_from_json(bucket, key, data) except: m.unlock() raise return data
def set_ins_string_object(bucket, key, string_data): """Set the value of 'key' in 'bucket' to the string 'string_data', if (and only if) this key has not already been set (ins = 'if not set'). This returns the object at this key after the operation (either the set string, or the value that was previously set) """ from Acquire.ObjectStore import Mutex as _Mutex m = _Mutex(bucket=bucket, key=key) try: val = ObjectStore.get_string_object(bucket, key) except: val = None if val is not None: m.unlock() return val else: try: ObjectStore.set_string_object(bucket, key, string_data) except: m.unlock() raise return string_data
def _generate_service_uid(bucket, registry_uid): """Function to generate a new service_uid on this registry. The UIDs have the form a0-a0, when "a" is any letter from [a-zA-Z] and "0" is any number from [0-9]. This give 520 possible values for each part either side of the hyphen. The part on the left of the hypen is the root UID, which matches the root of the service_uid of the registry service that registered this service (the service_uid of a registry service has the UID root-root). If more than 520 values are needed, then either side of the ID can be extended by additional pairs of a0 digits, using a "." to separate pairs, e.g. the service_uid for registry b4-b4 that comes after b4-Z9.Z9.Z9 is b4-a0.a0.a0.a0 similarly, the registry after Z9 is A0-A0. This means that a0.a0-a0.a0.a0.a0 would be a perfectly valid ID. We would only need IDs of this length if we have ~270k registry services, and this service_uid came from a service that had registered ~73 trillion services... The registry root Z9, with registry Z9-Z9 is reserved for the temporary registry created during testing """ from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.ObjectStore import Mutex as _Mutex root = registry_uid.split("-")[0] key = "%s/last_service_uid" % _registry_key mutex = _Mutex(key=key) try: last_vals = _ObjectStore.get_object_from_json(bucket=bucket, key=key) last_vals = _inc_uid(last_vals) except: last_vals = [0, 0] service_uid = "%s-%s" % (root, _to_uid(last_vals)) while service_uid == registry_uid: last_vals = _inc_uid(last_vals) service_uid = "%s-%s" % (root, _to_uid(last_vals)) _ObjectStore.set_object_from_json(bucket=bucket, key=key, data=last_vals) mutex.unlock() return service_uid
def load_test_and_set(uid, expected_state, new_state, bucket=None): """Static method to load up the Transaction record associated with the passed UID, check that the transaction state matches 'expected_state', and if it does, to update the transaction state to 'new_state'. This returns the loaded (and updated) transaction """ if bucket is None: bucket = _login_to_service_account() from ._ledger import Ledger as _Ledger try: mutex = _Mutex(uid, timeout=600, lease_time=600) except Exception as e: raise LedgerError("Cannot secure a Ledger mutex for transaction " "'%s'. Error = %s" % (uid, str(e))) try: transaction = _Ledger.load_transaction(uid, bucket) if transaction.transaction_state() != expected_state: raise TransactionError( "Cannot update the state of the transaction %s from " "%s to %s as it is not in the expected state" % (str(transaction), expected_state.value, new_state.value)) transaction._transaction_state = new_state except: mutex.unlock() raise # now need to write anything back if the state isn't changed if expected_state == new_state: return transaction # make sure we have enough time remaining on the lease to be # able to write this result back to the object store... if mutex.seconds_remaining_on_lease() < 100: try: mutex.fully_unlock() except: pass return TransactionRecord.load_test_and_set(uid, expected_state, new_state, bucket) try: _Ledger.save_transaction(transaction, bucket) except: mutex.unlock() raise return transaction
def _refresh_this_service_keys_and_certs(service_info, service_password): from Acquire.Service import Service as _Service service = _Service.from_data(service_info, service_password) if service._uid == "STAGE1": return service_info if not service.should_refresh_keys(): return service_info oldkeys = service.dump_keys(include_old_keys=False) # now write the old keys to storage from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.ObjectStore import Mutex as _Mutex from Acquire.Service import get_service_account_bucket as \ _get_service_account_bucket bucket = _get_service_account_bucket() key = "%s/oldkeys/%s" % (_service_key, oldkeys["datetime"]) _ObjectStore.set_object_from_json(bucket, key, oldkeys) # now write the pointers from fingerprint to file... for fingerprint in oldkeys.keys(): if fingerprint not in ["datetime", "encrypted_passphrase"]: _ObjectStore.set_string_object( bucket, "%s/oldkeys/fingerprints/%s" % (_service_key, fingerprint), key) # generate new keys last_update = service.last_key_update() service.refresh_keys() # now lock the object store so that we are the only function # that can write the new keys to global state m = _Mutex(key=service.uid(), bucket=bucket) service_data = _ObjectStore.get_object_from_json(bucket, _service_key) service_info = _Service.from_data(service_data) if service_info.last_key_update() == last_update: # no-one else has beaten us - write the updated keys to global state _ObjectStore.set_object_from_json(bucket, _service_key, service.to_data(service_password)) m.unlock() return service_data
def refresh_service_keys_and_certs(service, force_refresh=False): """This function will check if any key rotation is needed, and if so, it will automatically refresh the keys and certificates. The old keys and certificates will be stored in a database of old keys and certificates """ assert_running_service() if service._uid == "STAGE1": return service if (not force_refresh) and (not service.should_refresh_keys()): return service # ensure that the current keys are saved to the object store save_service_keys_to_objstore() # generate new keys last_update = service.last_key_update() service.refresh_keys() # now lock the object store so that we are the only function # that can write the new keys to global state from Acquire.Service import get_service_account_bucket as \ _get_service_account_bucket from Acquire.Service import Service as _Service from Acquire.ObjectStore import Mutex as _Mutex from Acquire.ObjectStore import ObjectStore as _ObjectStore bucket = _get_service_account_bucket() m = _Mutex(key=service.uid(), bucket=bucket) service_data = _ObjectStore.get_object_from_json(bucket, _service_key) service_info = _Service.from_data(service_data) if service_info.last_key_update() == last_update: # no-one else has beaten us - write the updated keys to global state _ObjectStore.set_object_from_json( bucket, _service_key, service.to_data(_get_service_password())) m.unlock() # clear the cache as we will need to load a new object clear_serviceinfo_cache() return get_this_service(need_private_access=True)
def create_account(self, name, description=None, overdraft_limit=None, bucket=None): """Create a new account called 'name' in this group. This will return the existing account if it already exists """ if name is None: raise ValueError("You must pass a name of the new account") account_key = self._account_key(name) if bucket is None: bucket = _login_to_service_account() try: account_uid = _ObjectStore.get_string_object(bucket, account_key) except: account_uid = None if account_uid is not None: # this account already exists - just return it account = _Account(uid=account_uid, bucket=bucket) if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) return account # make sure that no-one has created this account before m = _Mutex(account_key, timeout=600, lease_time=600, bucket=bucket) try: account_uid = _ObjectStore.get_string_object(bucket, account_key) except: account_uid = None if account_uid is not None: m.unlock() # this account already exists - just return it account = _Account(uid=account_uid, bucket=bucket) if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) return account # write a temporary UID to the object store so that we # can ensure we are the only function to create it try: _ObjectStore.set_string_object(bucket, account_key, "under_construction") except: m.unlock() raise m.unlock() # ok - we are the only function creating this account. Let's try # to create it properly try: account = _Account(name=name, description=description, bucket=bucket) except: try: _ObjectStore.delete_object(bucket, account_key) except: pass raise if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) _ObjectStore.set_string_object(bucket, account_key, account.uid()) return account
def add_admin_user(service, account_uid, authorisation=None): """Function that is called to add a new user account as a service administrator. If this is the first account then authorisation is not needed. If this is the second or subsequent admin account, then you need to provide an authorisation signed by one of the existing admin users. If you need to reset the admin users then delete the user accounts from the service. """ assert_running_service() from Acquire.Service import get_service_account_bucket as \ _get_service_account_bucket from Acquire.ObjectStore import Mutex as _Mutex from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.ObjectStore import get_datetime_now_to_string as \ _get_datetime_now_to_string bucket = _get_service_account_bucket() # see if the admin account details exists... admin_key = "%s/admin_users" % _service_key # ensure that we have exclusive access to this service mutex = _Mutex(key=admin_key, bucket=bucket) try: admin_users = _ObjectStore.get_object_from_json(bucket, admin_key) except: admin_users = None if admin_users is None: # this is the first admin user - automatically accept admin_users = {} authorised_by = "first admin" else: # validate that the new user has been authorised by an existing # admin... if authorisation is None: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "You must supply a valid authorisation from an existing admin " "user if you want to add a new admin user.") if authorisation.user_uid() not in admin_users: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "The authorisation for the new admin account is not valid " "because the user who signed it is not a valid admin on " "this service.") authorisation.verify(account_uid) authorised_by = authorisation.user_uid() # everything is ok - add this admin user to the admin_users # dictionary - save the date and time they became an admin, # and how they achieved this status (first admin, or whoever # authorised them) admin_users[account_uid] = { "datetime": _get_datetime_now_to_string(), "authorised_by": authorised_by } # everything is done, so now write this data to the object store _ObjectStore.set_object_from_json(bucket, admin_key, _json.dumps(admin_users)) # we can (finally!) release the mutex, as everyone else should now # be able to see the account mutex.unlock() _cache_adminusers.clear()
def setup_this_service(service_type, canonical_url, registry_uid, username, password): """Call this function to setup a new service that will serve at 'canonical_url', will be of the specified service_type. This will be registered at the registry at UID registry_uid (1) Delete the object store value "_service" if you want to reset the actual Service. This will assign a new UID for the service which would reset the certificates and keys. This new service will need to be re-introduced to other services that need to trust it """ assert_running_service() from Acquire.Service import get_service_account_bucket as \ _get_service_account_bucket from Acquire.ObjectStore import Mutex as _Mutex from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import Service as _Service bucket = _get_service_account_bucket() # ensure that this is the only time the service is set up mutex = _Mutex(key=_service_key, bucket=bucket, lease_time=120) try: service_info = _ObjectStore.get_object_from_json(bucket, _service_key) except: service_info = None service = None service_password = _get_service_password() user_uid = None otp = None if service_info: try: service = _Service.from_data(service_info, service_password) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Something went wrong reading the Service data. You should " "either debug the error or delete the data at key '%s' " "to allow the service to be reset and constructed again. " "The error was %s" % (_service_key, str(e))) if service.uid().startswith("STAGE1"): from Acquire.Service import ServiceAccountError raise ServiceAccountError( "The service is currently under construction. Please " "try again later...") if service is None: # we need to create the new service if (service_type is None) or (canonical_url is None): from Acquire.Service import ServiceAccountError raise ServiceAccountError( "You need to supply both the service_type and canonical_url " "in order to initialise a new Service") # we need to build the service account - first stage 1 service = _Service.create(service_url=canonical_url, service_type=service_type) # write the stage1 service data, encrypted using the service password. # This will be needed to answer the challenge from the registry service_data = service.to_data(service_password) _ObjectStore.set_object_from_json(bucket, _service_key, service_data) # now we can register the service with a registry - this # will return the stage2-constructed service from Acquire.Registry import register_service as _register_service service = _register_service(service=service, registry_uid=registry_uid) canonical_url = _Service.get_canonical_url(canonical_url) if service.service_type() != service_type or \ service.canonical_url() != canonical_url: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "The existing service has a different type or URL to that " "requested at setup. The request type and URL are %s and %s, " "while the actual service type and URL are %s and %s." % (service_type, canonical_url, service.service_type(), service.canonical_url())) # we can add the first admin user service_uid = service.uid() skelkey = service.skeleton_key().public_key() # now register the new admin user account - remembering to # encode the password from Acquire.Client import Credentials as _Credentials password = _Credentials.encode_password(password=password, identity_uid=service_uid) from Acquire.Identity import UserAccount as _UserAccount (user_uid, otp) = _UserAccount.create(username=username, password=password, _service_uid=service_uid, _service_public_key=skelkey) add_admin_user(service, user_uid) # write the service data, encrypted using the service password service_data = service.to_data(service_password) # reload the data to check it is ok, and also to set the right class service = _Service.from_data(service_data, service_password) # now it is ok, save this data to the object store _ObjectStore.set_object_from_json(bucket, _service_key, service_data) mutex.unlock() from Acquire.Service import clear_service_cache as _clear_service_cache _clear_service_cache() return (service, user_uid, otp)
def create_account(self, name, description=None, overdraft_limit=None, bucket=None, authorisation=None): """Create a new account called 'name' in this group. This will return the existing account if it already exists Args: name (str): Name of account to create description (default=None): Description of account overdraft_limit (int, default=None): Limit of overdraft bucket (dict, default=None): Bucket to load data from Returns: Account: New Account object """ if name is None: raise ValueError("You must pass a name of the new account") from Acquire.Identity import Authorisation as _Authorisation if authorisation is not None: if not isinstance(authorisation, _Authorisation): raise TypeError("The authorisation must be type Authorisation") if self._user_guid is not None: if self._user_guid != authorisation.user_guid(): raise PermissionError( "The same user who opened this accounts group (%s) " "must create accounts in this group (%s)" % (self._user_guid, authorisation.user_guid())) authorisation.verify("create_account %s" % name) self._assert_is_writeable() account_key = self._account_key(name) if bucket is None: from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket bucket = _get_service_account_bucket() from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Accounting import Account as _Account try: account_uid = _ObjectStore.get_string_object(bucket, account_key) except: account_uid = None if account_uid is not None: # this account already exists - just return it account = _Account(uid=account_uid, bucket=bucket) if account.group_name() != self.name(): account.set_group(self) if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) return account # make sure that no-one has created this account before from Acquire.ObjectStore import Mutex as _Mutex m = _Mutex(account_key, timeout=600, lease_time=600, bucket=bucket) try: account_uid = _ObjectStore.get_string_object(bucket, account_key) except: account_uid = None if account_uid is not None: m.unlock() # this account already exists - just return it account = _Account(uid=account_uid, bucket=bucket) if account.group_name() != self.name(): account.set_group(self) if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) return account # write a temporary UID to the object store so that we # can ensure we are the only function to create it try: _ObjectStore.set_string_object(bucket, account_key, "under_construction") except: m.unlock() raise m.unlock() # ok - we are the only function creating this account. Let's try # to create it properly. The account is created with the same # ACLRules of the group. try: from Acquire.Identity import ACLRules as _ACLRules account = _Account(name=name, description=description, group_name=self.name(), aclrules=_ACLRules.inherit(), bucket=bucket) except: try: _ObjectStore.delete_object(bucket, account_key) except: pass raise if overdraft_limit is not None: account.set_overdraft_limit(overdraft_limit, bucket=bucket) _ObjectStore.set_string_object(bucket, account_key, account.uid()) return account