def _unlock_userinfo(self, userinfo): """Function used to unlock (decrypt) the passed userinfo""" from Acquire.ObjectStore import string_to_bytes as _string_to_bytes import json as _json key = self._get_wallet_key() data = _string_to_bytes(userinfo["data"]) result = _json.loads(key.decrypt(data)) result["user_uid"] = userinfo["user_uid"] return result
def get_public_certs(identity_url, username, session_uid): """Call the identity_url to obtain the public keys and certificates of the user with 'username' logged in using the specified session_uid """ response = _call_function(identity_url, "get_keys", username=username, session_uid=session_uid) public_key = None public_cert = None if "public_key" in response: public_key = _PublicKey.read_bytes( _string_to_bytes(response["public_key"])) if "public_cert" in response: public_cert = _PublicKey.read_bytes( _string_to_bytes(response["public_cert"])) return (public_key, public_cert)
def decrypt_data(self, data): """Decrypt the pased data - this will only work in the daemon running on the cluster itself. The data must have been encrypted using the 'encrypt_data' function of this object. """ if self.is_null(): raise PermissionError("You cannot decrypt using a null cluster!") import json as _json if isinstance(data, str): data = _json.loads(data) try: from Acquire.ObjectStore import string_to_bytes as _string_to_bytes cluster_uid = data["cluster_uid"] fingerprint = data["fingerprint"] data = _string_to_bytes(data["encrypted_data"]) except Exception as e: from Acquire.Crypto import DecryptionError raise DecryptionError( "The encrypted data is not of the correct format: %s" % str(e)) if cluster_uid != self.uid(): from Acquire.Crypto import DecryptionError raise DecryptionError( "Cannot decrypt the data as it wasn't encrypted for this " "cluster - unmatched cluster UID: %s versus %s" % (cluster_uid, self.uid())) key = self.private_key() try: i = len(self._oldkeys) except: i = 0 while fingerprint != key.fingerprint(): i = i-1 if i < 0: from Acquire.Crypto import DecryptionError raise DecryptionError( "Cannot decrypt the data as we don't recognise the " "fingerprint of the encryption key: %s" % fingerprint) key = self._oldkeys[i] data = key.decrypt(data) return _json.loads(data)
def from_data(data): """Return an authorisation created from the json-decoded dictionary""" auth = Authorisation() if (data and len(data) > 0): auth._user_uid = data["user_uid"] auth._session_uid = data["session_uid"] auth._identity_url = data["identity_url"] auth._auth_timestamp = data["auth_timestamp"] auth._signature = _string_to_bytes(data["signature"]) auth._last_validated_time = None if "is_testing" in data: auth._is_testing = data["is_testing"] return auth
def from_data(data, passphrase=None): """Return a OSPar constructed from the passed json-deserliased dictionary Args: data (dict): JSON-deserialised dictionary from which to create OSPar Returns: OSPar: OSPar object created from dict """ if data is None or len(data) == 0: return OSPar() from Acquire.ObjectStore import string_to_datetime \ as _string_to_datetime from Acquire.ObjectStore import string_to_bytes \ as _string_to_bytes par = OSPar() par._url = _string_to_bytes(data["url"]) par._key = data["key"] par._uid = data["uid"] if par._key is not None: par._key = str(par._key) par._expires_datetime = _string_to_datetime(data["expires_datetime"]) par._is_readable = data["is_readable"] par._is_writeable = data["is_writeable"] if "service_url" in data: par._service_url = data["service_url"] if "privkey" in data: if passphrase is not None: from Acquire.Crypto import PrivateKey as _PrivateKey par._privkey = _PrivateKey.from_data(data["privkey"], passphrase) # note that we don't load the driver details as this # is stored and loaded separately on the service return par
def _get_key(key, fingerprint=None): """The user may pass the key in multiple ways. It could just be a key. Or it could be a function that gets the key on demand. Or it could be a dictionary that has the key stored under "encryption_public_key" """ from Acquire.Crypto import PublicKey as _PublicKey from Acquire.Crypto import PrivateKey as _PrivateKey if key is None: return None elif isinstance(key, _PublicKey) or isinstance(key, _PrivateKey): key = key elif isinstance(key, dict): try: key = key["encryption_public_key"] except: key = None if key is not None: from Acquire.ObjectStore import string_to_bytes as _string_to_bytes key = _PublicKey.read_bytes(_string_to_bytes(key)) else: key = key(fingerprint=fingerprint) if fingerprint is not None: if key is None: from Acquire.Crypto import KeyManipulationError raise KeyManipulationError( "Cannot find the key with fingerprint %s!" % fingerprint) elif key.fingerprint() != fingerprint: from Acquire.Crypto import KeyManipulationError raise KeyManipulationError( "Cannot find a key with the required fingerprint (%s). " "The only key has fingerprint %s" % (fingerprint, key.fingerprint())) return key
def _get_key(key): """The user may pass the key in multiple ways. It could just be a key. Or it could be a function that gets the key on demand. Or it could be a dictionary that has the key stored under "encryption_public_key" """ if key is None: return None elif isinstance(key, _PublicKey): return key elif isinstance(key, dict): try: key = key["encryption_public_key"] except: return None key = _PublicKey.read_bytes(_string_to_bytes(key)) else: try: key = key() except: pass return key
def from_data(data): """Return an object created from the passed json-deserialised dictionary. Note that this does not contain any information about the local file itself - just the name it should be called on the object store and the size, checksum and acl. If the file (or compressed file) is sufficiently small then this will also contain the packed version of that file data Args: data (dict): JSON-deserialised dictionary Returns: FileHandle: FileHandle object created from dictionary """ f = FileHandle() if data is not None and len(data) > 0: from Acquire.Storage import ACLRule as _ACLRule f._filename = data["filename"] f._filesize = int(data["filesize"]) f._checksum = data["checksum"] f._drive_uid = data["drive_uid"] if "compression" in data: f._compression = data["compression"] if "aclrules" in data: from Acquire.Storage import ACLRules as _ACLRules f._aclrules = _ACLRules.from_data(data["aclrules"]) if "filedata" in data: from Acquire.ObjectStore import string_to_bytes \ as _string_to_bytes f._local_filedata = _string_to_bytes(data["filedata"]) return f
def unpack_arguments(args, key=None, public_cert=None, is_return_value=False, function=None, service=None): """Call this to unpack the passed arguments that have been encoded as a json string, packed using pack_arguments. If is_return_value is True, then this will simply return the unpacked return valu Otherwise, this will return a tuple containing (function, args, keys) where function is the name of the function to be called, args are the arguments to the function, and keys is a dictionary that may contain keys or additional instructions that will be used to package up the return value from calling the function. This function is also called as unpack_return_value, in which case is_return_value is set as True, and only the dictionary is returned. The 'function' on 'service' that was called (or to be called) can also be passed. These are used to help provide more context for error messages. Args: args (str) : should be a JSON encoded UTF-8 """ if not (args and len(args) > 0): if is_return_value: return None else: return (None, None, None) # args should be a json-encoded utf-8 string try: data = _json.loads(args) except Exception as e: from Acquire.Service import UnpackingError raise UnpackingError("Cannot decode json from '%s' : %s" % (data, str(e))) while not isinstance(data, dict): if not (data and len(data) > 0): if is_return_value: return None else: return (None, None, None) try: data = _json.loads(data) except Exception as e: from Acquire.Service import UnpackingError raise UnpackingError( "Cannot decode a json dictionary from '%s' : %s" % (data, str(e))) if "payload" in data: payload = data["payload"] else: payload = None if is_return_value and payload is not None: # extra checks if this is a return value of a function rather # than the arguments if len(payload) == 1 and "error" in payload: from Acquire.Service import RemoteFunctionCallError raise RemoteFunctionCallError( "Calling %s on %s resulted in error: '%s'" % (function, service, payload["error"])) if "status" in payload: if payload["status"] != 0: if "exception" in payload: _unpack_and_raise(function, service, payload["exception"]) else: from Acquire.Service import RemoteFunctionCallError raise RemoteFunctionCallError( "Calling %s on %s exited with status %d: %s" % (function, service, payload["status"], payload)) try: is_encrypted = data["encrypted"] except: is_encrypted = False from Acquire.ObjectStore import string_to_bytes as _string_to_bytes if public_cert: if not is_encrypted: from Acquire.Service import UnpackingError raise UnpackingError( "Cannot unpack the result of %s on %s as it should be " "signed, but it isn't! (only encrypted results are signed) " "Response == %s" % (function, service, _json.dumps(data))) try: signature = _string_to_bytes(data["signature"]) except: signature = None if signature is None: from Acquire.Service import UnpackingError raise UnpackingError( "We requested that the data was signed " "when calling %s on %s, but a signature was not provided!" % (function, service)) if is_encrypted: encrypted_data = _string_to_bytes(data["data"]) try: fingerprint = data["fingerprint"] except: fingerprint = None if public_cert: try: public_cert.verify(signature, encrypted_data) except Exception as e: raise UnpackingError("The signature of the returned data " "from calling %s on %s " "is incorrect and does not match what we " "know! %s" % (function, service, str(e))) decrypted_data = _get_key(key, fingerprint).decrypt(encrypted_data) return unpack_arguments(decrypted_data, is_return_value=is_return_value, function=function, service=service) if payload is None: from Acquire.Service import UnpackingError raise UnpackingError( "We should have been able to extract the payload from " "%s" % data) if is_return_value: try: return payload["return"] except: # no return value from this function return None else: try: function = data["function"] except: function = None return (function, payload, data)
def get_service_account_bucket(testing_dir=None): """This function logs into the object store account of the service account. Accessing the object store means being able to access all resources and which can authorise the creation of access all resources on the object store. Obviously this is a powerful account, so only log into it if you need it!!! The login information should not be put into a public repository or stored in plain text. In this case, the login information is held in an environment variable (which should be encrypted or hidden in some way...) """ from Acquire.Service import assert_running_service as \ _assert_running_service _assert_running_service() # read the password for the secret key from the filesystem try: with open("secret_key", "r") as FILE: password = FILE.readline()[0:-1] except: password = None # we must be in testing mode... from Acquire.ObjectStore import use_testing_object_store_backend as \ _use_testing_object_store_backend # see if this is running in testing mode... global _current_testing_objstore if testing_dir: _current_testing_objstore = testing_dir return _use_testing_object_store_backend(testing_dir) elif _current_testing_objstore: return _use_testing_object_store_backend(_current_testing_objstore) if password is None: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "You need to supply login credentials via the 'secret_key' " "file, and 'SECRET_KEY' and 'SECRET_CONFIG' environment " "variables! %s" % testing_dir) secret_key = _os.getenv("SECRET_KEY") if secret_key is None: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "You must supply the password used to unlock the configuration " "key in the 'SECRET_KEY' environment variable") try: secret_key = _json.loads(secret_key) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Unable to decode valid JSON from the secret key: %s" % str(e)) # use the password to decrypt the SECRET_KEY in the config try: from Acquire.Crypto import PrivateKey as _PrivateKey secret_key = _PrivateKey.from_data(secret_key, password) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Unable to open the private SECRET_KEY using the password " "supplied in the 'secret_key' file: %s" % str(e)) config = _os.getenv("SECRET_CONFIG") if config is None: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "You must supply the encrypted config in teh 'SECRET_CONFIG' " "environment variable!") try: from Acquire.ObjectStore import string_to_bytes as _string_to_bytes config = secret_key.decrypt(_string_to_bytes(config)) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Cannot decrypt the 'SECRET_CONFIG' with the 'SECRET_KEY'. Are " "you sure that the configuration has been set up correctly? %s " % str(e)) # use the secret_key to decrypt the config in SECRET_CONFIG try: config = _json.loads(config) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Unable to decode valid JSON from the config: %s" % str(e)) # get info from this config access_data = config["LOGIN"] bucket_data = config["BUCKET"] # save the service password to the environment _os.environ["SERVICE_PASSWORD"] = config["PASSWORD"] # save any other decrypted config data to environment variables for key in config.keys(): if key not in ["LOGIN", "BUCKET", "PASSWORD"]: _os.environ[key] = config[key] # we have OCI login details, so make sure that we are using # the OCI object store backend from Acquire.ObjectStore import use_oci_object_store_backend as \ _use_oci_object_store_backend _use_oci_object_store_backend() # now login and create/load the bucket for this account try: from ._oci_account import OCIAccount as _OCIAccount account_bucket = _OCIAccount.create_and_connect_to_bucket( access_data, bucket_data["compartment"], bucket_data["bucket"]) except Exception as e: from Acquire.Service import ServiceAccountError raise ServiceAccountError( "Error connecting to the service account: %s" % str(e)) return account_bucket
def download_next_chunk(self): """Download the next chunk. Returns 'True' if something was downloaded, else it returns 'False' """ if not self.is_open(): return False service = self.service() if service is None: raise PermissionError( "Cannot download a chunk from a null service!") from Acquire.Crypto import Hash as _Hash secret = _Hash.multi_md5( self._secret, "%s%s%d" % (self._drive_uid, self._file_uid, self._next_index)) args = {} args["uid"] = self._uid args["drive_uid"] = self._drive_uid args["file_uid"] = self._file_uid args["chunk_index"] = self._next_index args["secret"] = secret response = service.call_function(function="download_chunk", args=args) if "meta" in response: import json as _json meta = _json.loads(response["meta"]) checksum = meta["checksum"] from Acquire.ObjectStore import string_to_bytes \ as _string_to_bytes chunk = _string_to_bytes(response["chunk"]) md5 = _Hash.md5(chunk) if checksum != md5: from Acquire.Storage import FileValidationError raise FileValidationError( "Problem downloading - checksums don't agree: %s vs %s" % (checksum, md5)) import bz2 as _bz2 chunk = _bz2.decompress(chunk) self._FILE.write(chunk) self._FILE.flush() chunk = None self._next_index = self._next_index + 1 if "num_chunks" in response: num_chunks = int(response["num_chunks"]) if self._next_index >= num_chunks: # nothing more to download self.close() return True
def unpack_arguments(args, key=None, public_cert=None): """Call this to unpack the passed arguments that have been encoded as a json string, packed using pack_arguments. This will always return a dictionary. If there are no arguments, then an empty dictionary will be returned. If 'public_cert' is supplied then a signature of the result will be verified using 'public_cert' """ if not (args and len(args) > 0): return {} # args should be a json-encoded utf-8 string try: data = _json.loads(args) except Exception as e: raise UnpackingError("Cannot decode json from '%s' : %s" % (data, str(e))) while not isinstance(data, dict): if not (data and len(data) > 0): return {} try: data = _json.loads(data) except Exception as e: raise UnpackingError( "Cannot decode a json dictionary from '%s' : %s" % (data, str(e))) if len(data) == 1 and "error" in data: raise RemoteFunctionCallError( "Server returned the error string: '%s'" % (data["error"])) try: is_encrypted = data["encrypted"] except: if public_cert: raise UnpackingError( "Cannot unpack the result as it should be " "signed, but it isn't! (only encrypted results are signed)") is_encrypted = False if public_cert: try: signature = _string_to_bytes(data["signature"]) except: raise UnpackingError("We requested that the data was signed " "but a signature was not provided!") if is_encrypted: encrypted_data = _string_to_bytes(data["data"]) if public_cert: try: public_cert.verify(signature, encrypted_data) except Exception as e: raise UnpackingError("The signature of the returned data " "is incorrect and does not match what we " "know! %s" % str(e)) decrypted_data = _get_key(key).decrypt(encrypted_data) return unpack_arguments(decrypted_data.decode("utf-8")) else: return data
def download(self, filename=None, version=None, dir=None, force_par=False): """Download this file into the local directory the local directory, or 'dir' if specified, calling the file 'filename' (or whatever it is called on the Drive if not specified). If a local file exists with this name, then a new, unique filename will be used. This returns the local filename of the downloaded file (with full absolute path) Note that this only downloads files for which you have read-access. If the file is not readable then an exception is raised and nothing is returned If 'version' is specified then download a specific version of the file. Otherwise download the version associated with this file object """ if self.is_null(): raise PermissionError("Cannot download a null File!") if self._creds is None: raise PermissionError("We have not properly opened the file!") if filename is None: filename = self._metadata.name() drive_uid = self._metadata.drive().uid() from Acquire.Client import create_new_file as \ _create_new_file if self._creds.is_user(): privkey = self._creds.user().session_key() else: from Acquire.Crypto import get_private_key as _get_private_key privkey = _get_private_key("parkey") args = {"drive_uid": drive_uid, "filename": self._metadata.name(), "encryption_key": privkey.public_key().to_data()} if self._creds.is_user(): from Acquire.Client import Authorisation as _Authorisation authorisation = _Authorisation( resource="download %s %s" % (drive_uid, self._metadata.name()), user=self._creds.user()) args["authorisation"] = authorisation.to_data() elif self._creds.is_par(): par = self._creds.par() par.assert_valid() args["par_uid"] = par.uid() args["secret"] = self._creds.secret() if force_par: args["force_par"] = True if version is not None: from Acquire.ObjectStore import datetime_to_string \ as _datetime_to_string args["version"] = _datetime_to_string(version) elif self._metadata.version() is not None: args["version"] = self._metadata.version() storage_service = self._creds.storage_service() response = storage_service.call_function( function="download", args=args) from Acquire.Client import FileMeta as _FileMeta filemeta = _FileMeta.from_data(response["filemeta"]) if "filedata" in response: # we have already downloaded the file to 'filedata' filedata = response["filedata"] from Acquire.ObjectStore import string_to_bytes \ as _string_to_bytes filedata = _string_to_bytes(response["filedata"]) del response["filedata"] # validate that the size and checksum are correct filemeta.assert_correct_data(filedata) if filemeta.is_compressed(): # uncompress the data from Acquire.Client import uncompress as _uncompress filedata = _uncompress( inputdata=filedata, compression_type=filemeta.compression_type()) # write the data to the specified local file... filename = _create_new_file(filename=filename, dir=dir) with open(filename, "wb") as FILE: FILE.write(filedata) FILE.flush() elif "download_par" in response: from Acquire.ObjectStore import OSPar as _OSPar filename = _create_new_file(filename=filename, dir=dir) par = _OSPar.from_data(response["download_par"]) par.read(privkey).get_object_as_file(filename) par.close(privkey) # validate that the size and checksum are correct filemeta.assert_correct_data(filename=filename) # uncompress the file if desired if filemeta.is_compressed(): from Acquire.Client import uncompress as _uncompress _uncompress(inputfile=filename, outputfile=filename, compression_type=filemeta.compression_type()) elif "downloader" in response: from Acquire.Client import ChunkDownloader as _ChunkDownloader downloader = _ChunkDownloader.from_data(response["downloader"], privkey=privkey, service=storage_service) filename = downloader.download(filename=filename, dir=dir) filemeta._copy_credentials(self._metadata) self._metadata = filemeta return filename
def validate_password(user_uid, username, device_uid, secrets, password, otpcode, remember_device): """Validate that the passed password and one-time-code are valid. If they are, then return a tuple of the UserAccount of the unlocked user, the OTP that is used to generate secrets, and the device_uid of the login device If 'remember_device' is True and 'device_uid' is None, then this creates a new OTP for the login device, which is returned, and a new device_uid for that device. The password needed to match this device is a MD5 of the normal user password. """ from Acquire.Crypto import PrivateKey as _PrivateKey from Acquire.Crypto import OTP as _OTP from Acquire.ObjectStore import string_to_bytes as _string_to_bytes privkey = _PrivateKey.from_data(data=secrets["private_key"], passphrase=password) # decrypt and validate the OTP code data = _string_to_bytes(secrets["otpsecret"]) otpsecret = privkey.decrypt(data) otp = _OTP(secret=otpsecret) otp.verify(code=otpcode, once_only=True) # everything is ok - we can load the user account via the # decrypted primary password primary_password = _string_to_bytes(secrets["primary_password"]) primary_password = privkey.decrypt(primary_password) from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket data = None secrets = None key = "%s/uids/%s" % (_user_root, user_uid) bucket = _get_service_account_bucket() try: data = _ObjectStore.get_object_from_json(bucket=bucket, key=key) except: pass if data is None: from Acquire.Identity import UserValidationError raise UserValidationError( "Unable to validate user as no account data is present!") from Acquire.Identity import UserAccount as _UserAccount user = _UserAccount.from_data(data=data, passphrase=primary_password) if user.uid() != user_uid: from Acquire.Identity import UserValidationError raise UserValidationError( "Unable to validate user as mismatch in user_uids!") if device_uid is None and remember_device: # create a new OTP that is unique for this device from Acquire.ObjectStore import create_uuid as _create_uuid from Acquire.Client import Credentials as _Credentials device_uid = _create_uuid() device_password = _Credentials.encode_device_uid( encoded_password=password, device_uid=device_uid) otp = UserCredentials.create(user_uid=user_uid, password=device_password, primary_password=primary_password, device_uid=device_uid) # now save a lookup so that we can find the user_uid from # the username and device-specific password encoded_password = UserCredentials.hash( username=username, password=device_password) key = "%s/passwords/%s/%s" % (_user_root, encoded_password, user_uid) from Acquire.ObjectStore import get_datetime_now_to_string \ as _get_datetime_now_to_string _ObjectStore.set_string_object( bucket=bucket, key=key, string_data=_get_datetime_now_to_string()) return {"user": user, "otp": otp, "device_uid": device_uid}
def login_to_service_account(testing_dir=None): """This function logs into the object store account of the service account. Accessing the object store means being able to access all resources and which can authorise the creation of access all resources on the object store. Obviously this is a powerful account, so only log into it if you need it!!! The login information should not be put into a public repository or stored in plain text. In this case, the login information is held in an environment variable (which should be encrypted or hidden in some way...) """ # read the password for the secret key from the filesystem try: with open("secret_key", "r") as FILE: password = FILE.readline()[0:-1] except: password = None # we must be in testing mode... from Acquire.ObjectStore import use_testing_object_store_backend as \ _use_testing_object_store_backend # see if this is running in testing mode... global _current_testing_objstore if testing_dir: _current_testing_objstore = testing_dir return _use_testing_object_store_backend(testing_dir) elif _current_testing_objstore: return _use_testing_object_store_backend(_current_testing_objstore) if password is None: raise ServiceAccountError( "You need to supply login credentials via the 'secret_key' " "file, and 'SECRET_KEY' and 'SECRET_CONFIG' environment " "variables! %s" % testing_dir) # use the password to decrypt the SECRET_KEY in the config secret_key = _PrivateKey.from_data(_json.loads(_os.getenv("SECRET_KEY")), password) # use the secret_key to decrypt the config in SECRET_CONFIG config = _json.loads( secret_key.decrypt(_string_to_bytes( _os.getenv("SECRET_CONFIG"))).decode("utf-8")) # get info from this config access_data = config["LOGIN"] bucket_data = config["BUCKET"] # save the service password to the environment _os.environ["SERVICE_PASSWORD"] = config["PASSWORD"] # we have OCI login details, so make sure that we are using # the OCI object store backend from Acquire.ObjectStore import use_oci_object_store_backend as \ _use_oci_object_store_backend _use_oci_object_store_backend() # now login and create/load the bucket for this account try: from ._oci_account import OCIAccount as _OCIAccount account_bucket = _OCIAccount.create_and_connect_to_bucket( access_data, bucket_data["compartment"], bucket_data["bucket"]) except Exception as e: raise ServiceAccountError( "Error connecting to the service account: %s" % str(e)) return account_bucket