class ClosureNameHolder(storagetypes.Object): ''' Mark a closure's name as in use ''' name = storagetypes.String() closure_id = storagetypes.Integer() required_attrs = ["name"] @classmethod def make_key_name(cls, name): return "ClosureNameHolder: name=%s" % (name) @classmethod def create_async(cls, _name, _id): return ClosureNameHolder.get_or_insert_async( ClosureNameHolder.make_key_name(_name), name=_name, closure_id=_id)
class VolumeNameHolder(storagetypes.Object): ''' Mark a Volume name as taken ''' name = storagetypes.String() volume_id = storagetypes.Integer() required_attrs = ["name"] @classmethod def make_key_name(cls, name): return "VolumeNameHolder: name=%s" % (name) @classmethod def create_async(cls, _name, _id): return VolumeNameHolder.get_or_insert_async( VolumeNameHolder.make_key_name(_name), name=_name, volume_id=_id)
class GatewayNameHolder(storagetypes.Object): ''' Mark a Gateway's name as in use ''' name = storagetypes.String() g_id = storagetypes.Integer() required_attrs = ["name"] @classmethod def make_key_name(cls, name): return "GatewayNameHolder: name=%s" % (name) @classmethod def create_async(cls, _name, _id): return GatewayNameHolder.get_or_insert_async( GatewayNameHolder.make_key_name(_name), name=_name, g_id=_id)
class SyndicateUserNameHolder(storagetypes.Object): ''' Mark a SyndicateUser email as taken ''' email = storagetypes.String() owner_id = storagetypes.Integer() required_attrs = ["email"] @classmethod def make_key_name(cls, email): return "SyndicateUserNameHolder: email=%s" % (email) @classmethod def create_async(cls, _email, _id): return SyndicateUserNameHolder.get_or_insert_async( SyndicateUserNameHolder.make_key_name(_email), email=_email, owner_id=_id)
class SyndicateUser(storagetypes.Object): USER_KEY_UNSET = "unset" USER_KEY_UNUSED = "unused" email = storagetypes.String() # used as the username owner_id = storagetypes.Integer() # UID field in Syndicate openid_url = storagetypes.Text() # OpenID identifying URL max_volumes = storagetypes.Integer( default=10 ) # how many Volumes can this user create? (-1 means unlimited) max_UGs = storagetypes.Integer( default=10) # how many UGs can this user create? max_RGs = storagetypes.Integer( default=10) # how many RGs can this user create? max_AGs = storagetypes.Integer( default=10) # how many AGs can this user create? max_requests = storagetypes.Integer( default=10) # how many pending Volume requests can this user create? is_admin = storagetypes.Boolean( default=False, indexed=False) # is this user an administrator? signing_public_key = storagetypes.Text( ) # PEM-encoded public key for authenticating this user, or USER_KEY_UNSET if it is not set, or USER_KEY_UNUSED if it will not be used signing_public_key_expiration = storagetypes.Integer( default=-1) # seconds since the epoch active = storagetypes.Boolean(default=False) # is this account active? allow_password_auth = storagetypes.Boolean( default=True) # allow password-based authentication? # one-time password for setting the signing public key activate_password_salt = storagetypes.String( ) # 32 bytes, but encoded as a hex string activate_password_hash = storagetypes.String() # SHA256 # for RPC key_type = "user" required_attrs = ["email", "openid_url", "signing_public_key_expiration"] key_attrs = ["email"] default_values = { "max_volumes": (lambda cls, attrs: 10), "max_UGs": (lambda cls, attrs: 10), "max_RGs": (lambda cls, attrs: 10), "max_AGs": (lambda cls, attrs: 10), "is_admin": (lambda cls, attrs: False), "openid_url": (lambda cls, attrs: ""), "signing_public_key_expiration": (lambda cls, attrs: -1), "active": (lambda cls, attrs: False), "allow_password_auth": (lambda cls, attrs: True) } validators = { "email": (lambda cls, value: valid_email(cls, value)), "signing_public_key": (lambda cls, value: not cls.is_signing_public_key_set(value) or cls. is_valid_key(value, USER_RSA_KEYSIZE)), "openid_url": (lambda cls, value: len(value) < 4096), # not much of a check here... "activate_password_salt": (lambda cls, value: len( str(value).translate(None, "0123456789abcdefABCDEF")) == 0 and len( str(value)) == 64), # 32-byte salt, encoded as a hex number "activate_password_hash": (lambda cls, value: len(str(value).translate(None, "0123456789abcdefABCDEF") ) == 0 and len(str(value)) == 64 ) # SHA256: 32-byte hash, encoded as a hex number } read_attrs_api_required = [ "email", "owner_id", "openid_url", "max_volumes", "max_UGs", "max_RGs", "max_AGs", "signing_public_key", "signing_public_key_expiration", ] read_attrs = read_attrs_api_required write_attrs_api_required = [ "openid_url", "signing_public_key", "allow_password_auth" ] write_attrs_admin_required = [ "max_volumes", "max_UGs", "max_RGs", "max_AGs", "is_admin" ] write_attrs = write_attrs_api_required + write_attrs_admin_required def owned_by(self, user): return user.owner_id == self.owner_id @classmethod def Authenticate(cls, email, data, data_signature): """ Authenticate a user via public-key cryptography. Verify that data was signed by the user's private key, given the signature and data. (use RSA PSS for security). Return the user on success; False on authentication error; None if the user doesn't exist """ user = SyndicateUser.Read(email) if user == None: return None if not SyndicateUser.is_signing_public_key_set( user.signing_public_key): logging.error("Key for %s is not set or unused" % email) return None ret = cls.auth_verify(user.signing_public_key, data, data_signature) if not ret: logging.error("Verification failed for %s" % email) return False else: return user def makeCert(self): ret = {} ret['expires'] = self.signing_public_key_expiration ret['pubkey'] = self.signing_public_key ret['email'] = self.email ret['openid_url'] = self.openid_url return ret @classmethod def Create(cls, email, **kwargs): """ Create a SyndicateUser. Required arguments: email -- Email address of the user. Serves as the username (str) openid_url -- OpenID identifier for authenticating this user (str) """ kwargs['email'] = email # sanity check SyndicateUser.fill_defaults(kwargs) # if we're given a signing public key, then set it. # otherwise, use the given salted password hash. skip_verify = [] if kwargs.has_key('activate_password_hash') and kwargs.has_key( 'activate_password_salt'): # don't check for this skip_verify = ['signing_public_key'] kwargs['signing_public_key'] = cls.USER_KEY_UNSET elif kwargs.has_key('signing_public_key'): # this had better be a valid key if not SyndicateUser.validators['signing_public_key']( SyndicateUser, kwargs['signing_public_key']): raise Exception("Invalid field: %s" % 'signing_public_key') # don't check for password hash and salt skip_verify = ['activate_password_hash', 'activate_password_salt'] else: # need either of these... raise Exception( "Need either signing_public_key or (activate_password_hash, activate_password_salt)" ) missing = SyndicateUser.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) invalid = SyndicateUser.validate_fields(kwargs, skip=skip_verify) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) user_key_name = SyndicateUser.make_key_name(email=email) user = storagetypes.memcache.get(user_key_name) if user == None: user_key = storagetypes.make_key(SyndicateUser, user_key_name) user = user_key.get() if user == None: # do not allow admin privileges kwargs['is_admin'] = False kwargs['owner_id'] = random.randint(1, 2**63 - 1) user_key_name = SyndicateUser.make_key_name(email=email) user = SyndicateUser.get_or_insert(user_key_name, **kwargs) # check for collisions if user.owner_id != kwargs['owner_id']: # collision raise Exception("User '%s' already exists" % email) return user.key else: raise Exception("User '%s' already exists" % email) else: raise Exception("User '%s' already exists" % email) @classmethod def CreateAdmin(cls, email, openid_url, signing_public_key, activate_password): """ Create the Admin user. NOTE: this will be called repeatedly, so use memcache """ user_key_name = SyndicateUser.make_key_name(email=email) user = storagetypes.memcache.get(user_key_name) if user == None: user_key = storagetypes.make_key(SyndicateUser, user_key_name) user = user_key.get() if user == None: # admin does not exist attrs = {} logging.info("Generating admin '%s'" % email) # fill defaults SyndicateUser.fill_defaults(attrs) attrs['email'] = email attrs['openid_url'] = openid_url attrs['owner_id'] = random.randint(1, 2**63 - 1) attrs['is_admin'] = True # generate password hash and salt import common.api as api pw_salt = api.password_salt() pw_hash = api.hash_password(activate_password, pw_salt) attrs['activate_password_hash'] = pw_hash attrs['activate_password_salt'] = pw_salt # possible that we haven't set the public key yet if not signing_public_key or len(signing_public_key) == 0: signing_public_key = cls.USER_KEY_UNSET attrs['signing_public_key'] = signing_public_key invalid = SyndicateUser.validate_fields(attrs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) user = SyndicateUser.get_or_insert(user_key_name, **attrs) # check for collisions if user.owner_id != attrs['owner_id']: # collision logging.warning("Admin '%s' already exists" % email) storagetypes.memcache.set(user_key_name, user) return user.key @classmethod def Read(cls, email_or_owner_id, async=False): """ Read a SyndicateUser Arguments: email_or_owner_id -- Email address of the user to read, or the owner ID (str or int) """ owner_id = None email = None try: owner_id = int(email_or_owner_id) except: email = email_or_owner_id if owner_id != None: return cls.Read_ByOwnerID(owner_id, async=async) user_key_name = SyndicateUser.make_key_name(email=email) user_key = storagetypes.make_key(SyndicateUser, user_key_name) user = storagetypes.memcache.get(user_key_name) if user == None: if async: return user_key.get_async(use_memcache=False) else: user = user_key.get(use_memcache=False) if not user: return None else: storagetypes.memcache.set(user_key_name, user) elif async: user = storagetypes.FutureWrapper(user) return user
class VolumeAccessRequest(storagetypes.Object): """ This object controls what kinds of Gateways a user can create within a Volume, and what capabilities they are allowed to have. """ STATUS_PENDING = 1 STATUS_GRANTED = 2 requester_owner_id = storagetypes.Integer() # owner id of the requester request_message = storagetypes.Text() # message to the owner volume_id = storagetypes.Integer() # ID of volume to join gateway_caps = storagetypes.Integer( indexed=False ) # gateway capabilities requested (only apply to User Gateways) nonce = storagetypes.Integer( indexed=False) # detect collision with another one of these request_timestamp = storagetypes.Integer() # when was the request made? status = storagetypes.Integer() # granted or pending? allowed_gateways = storagetypes.Integer( default=0) # bit vector representing GATEWAY_TYPE_*G (from msconfig) # purely for readability volume_name = storagetypes.String() required_attrs = [ "requester_owner_id", "nonce", "request_timestamp", "status", "volume_name", "volume_id" ] read_attrs = [ "requester_owner_id", "request_message", "volume_id", "gateway_caps", "request_timestamp", "status", "volume_name", "allowed_gateways" ] read_attrs_api_required = read_attrs def owned_by(self, user): return (user.owner_id == self.requester_owner_id) @classmethod def make_key_name(cls, requester_owner_id, volume_id): return "VolumeAccessRequest: owner_id=%s,volume_id=%s" % ( requester_owner_id, volume_id) @classmethod def create_async(cls, _requester_owner_id, _volume_id, _volume_name, _nonce, _status, **attrs): ts = int(storagetypes.get_time()) return VolumeAccessRequest.get_or_insert_async( VolumeAccessRequest.make_key_name(_requester_owner_id, _volume_id), requester_owner_id=_requester_owner_id, volume_id=_volume_id, nonce=_nonce, request_timestamp=ts, status=_status, volume_name=_volume_name, **attrs) @classmethod def RequestAccess(cls, owner_id, volume_id, volume_name, allowed_gateways, gateway_caps, message): """ Create a request that a particular user be allowed to provision Gateways for a particular Volume. If User Gateways are allowed, then gateway_caps controls what those capabilities are allowed to be. Include a message that the Volume owner will be able to read. Return if the request was successfully placed. Raise an exception if there is already a pending request. """ nonce = random.randint(-2**63, 2**63 - 1) req_fut = VolumeAccessRequest.create_async( owner_id, volume_id, volume_name, nonce, VolumeAccessRequest.STATUS_PENDING, request_message=message, gateway_caps=gateway_caps, allowed_gateways=allowed_gateways) req = req_fut.get_result() # duplicate? if req.nonce != nonce: raise Exception("User already attempted to join Volume '%s'" % (owner_id, volume_name)) return True @classmethod def GrantAccess(cls, owner_id, volume_id, volume_name, allowed_gateways=None, gateway_caps=None): """ Allow a given user to create Gateways within a given Volume, subject to given capabilities. """ # verify the arguments are valid if allowed_gateways is not None and (allowed_gateways & ~( (1 << GATEWAY_TYPE_UG) | (1 << GATEWAY_TYPE_RG) | (1 << GATEWAY_TYPE_AG))) != 0: # extra bits raise Exception("Invalid bit field for allowed_gateways (%x)" % (allowed_gateways)) if gateway_caps is not None and ( gateway_caps & ~(GATEWAY_CAP_READ_DATA | GATEWAY_CAP_READ_METADATA | GATEWAY_CAP_WRITE_DATA | GATEWAY_CAP_WRITE_METADATA | GATEWAY_CAP_COORDINATE)) != 0: # extra bits raise Exception("Invalid bit field for gateway_caps (%x)" % (gateway_caps)) nonce = random.randint(-2**63, 2**63 - 1) req_fut = VolumeAccessRequest.create_async( owner_id, volume_id, volume_name, nonce, VolumeAccessRequest.STATUS_GRANTED, request_message="", gateway_caps=gateway_caps, allowed_gateways=allowed_gateways) req = req_fut.get_result() if req.nonce != nonce: # Request existed. update and put if gateway_caps != None: req.gateway_caps = gateway_caps if allowed_gateways != None: req.allowed_gateways = allowed_gateways req.status = VolumeAccessRequest.STATUS_GRANTED req.put() req_key_name = VolumeAccessRequest.make_key_name(owner_id, volume_id) storagetypes.memcache.delete(req_key_name) return True @classmethod def GetAccess(cls, owner_id, volume_id): """ Get the access status of a user in a Volume. """ req_key_name = VolumeAccessRequest.make_key_name(owner_id, volume_id) req = storagetypes.memcache.get(req_key_name) if req != None: return req req_key = storagetypes.make_key(VolumeAccessRequest, req_key_name) req = req_key.get() if req != None: storagetypes.memcache.set(req_key_name, req) return req @classmethod def RemoveAccessRequest(cls, owner_id, volume_id): """ Delete an access request. """ req_key_name = VolumeAccessRequest.make_key_name(owner_id, volume_id) req_key = storagetypes.make_key(VolumeAccessRequest, req_key_name) storagetypes.deferred.defer(cls.delete_all, [req_key]) storagetypes.memcache.delete(req_key_name) return True @classmethod def ListUserAccessRequests(cls, owner_id, **q_opts): return VolumeAccessRequest.ListAll( {"VolumeAccessRequest.requester_owner_id ==": owner_id}, **q_opts) @classmethod def ListVolumeAccessRequests(cls, volume_id, **q_opts): return VolumeAccessRequest.ListAll( { "VolumeAccessRequest.volume_id ==": volume_id, "VolumeAccessRequest.status ==": VolumeAccessRequest.STATUS_PENDING }, **q_opts) @classmethod def ListVolumeAccess(cls, volume_id, **q_opts): return VolumeAccessRequest.ListAll( { "VolumeAccessRequest.volume_id ==": volume_id, "VolumeAccessRequest.status ==": VolumeAccessRequest.STATUS_GRANTED }, **q_opts) @classmethod def DeleteAccessRequestsByVolume(cls, volume_id, async=False): def __delete_var(var): req_key_name = VolumeAccessRequest.make_key_name( var.requester_owner_id, volume_id) storagetypes.memcache.delete(req_key_name) var.key.delete() return VolumeAccessRequest.ListAll( {"VolumeAccessRequest.volume_id ==": volume_id}, map_func=__delete_var, projection=['requester_owner_id'], async=async)
class Volume(storagetypes.Object): name = storagetypes.String() blocksize = storagetypes.Integer(indexed=False) # Stored in bytes!! active = storagetypes.Boolean() description = storagetypes.Text() owner_id = storagetypes.Integer() volume_id = storagetypes.Integer() version = storagetypes.Integer( indexed=False) # version of this Volume's metadata cert_version = storagetypes.Integer( indexed=False) # certificate bundle version private = storagetypes.Boolean( ) # if True, then this Volume won't be listed archive = storagetypes.Boolean( ) # only an authenticated AG owned by the same user that owns this Volume can write to this Volume allow_anon = storagetypes.Boolean( ) # if True, then anonymous users can access this Volume (i.e. users who don't have to log in) num_shards = storagetypes.Integer( default=20, indexed=False) # number of shards per entry in this volume metadata_public_key = storagetypes.Text( ) # Volume public key, in PEM format, for verifying metadata metadata_private_key = storagetypes.Text( ) # Volume private key, in PEM format, for signing metadata file_quota = storagetypes.Integer( ) # maximum number of files allowed here (-1 means unlimited) deleted = storagetypes.Boolean() # is this Volume deleted? default_gateway_caps = storagetypes.Integer(indexed=False) closure = storagetypes.Text( ) # base64-encoded closure for connecting to the cache providers # for RPC key_type = "volume" @classmethod def generate_metadata_keys(cls): """ Generate metadata public/private keys for metadata sign/verify """ return cls.generate_keys(VOLUME_RSA_KEYSIZE) required_attrs = [ "name", "blocksize", "owner_id", "private", "metadata_private_key", "default_gateway_caps", ] key_attrs = ["volume_id"] validators = { "name": (lambda cls, value: len( unicode(value).translate( dict( (ord(char), None) for char in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-. " ))) == 0 and not is_int(value)), "metadata_public_key": (lambda cls, value: cls.is_valid_key(value, VOLUME_RSA_KEYSIZE)), "metadata_private_key": (lambda cls, value: cls.is_valid_key(value, VOLUME_RSA_KEYSIZE)) } default_values = { "blocksize": (lambda cls, attrs: 61440), # 60 KB "version": (lambda cls, attrs: 1), "cert_version": (lambda cls, attrs: 1), "private": (lambda cls, attrs: True), "archive": (lambda cls, attrs: False), "allow_anon": (lambda cls, attrs: False), "active": (lambda cls, attrs: True), "file_quota": (lambda cls, attrs: -1), "deleted": (lambda cls, attrs: False), "num_shards": (lambda cls, attrs: 20), "default_gateway_caps": (lambda cls, attrs: GATEWAY_CAP_READ_METADATA | GATEWAY_CAP_READ_DATA ) # read only } read_attrs_api_required = [ "blocksize", "active", "version", "cert_version", "private", "archive", "allow_anon", "file_quota", "default_gateway_caps", "closure" ] read_attrs = [ "name", "description", "owner_id", "volume_id", "metadata_public_key", ] + read_attrs_api_required write_attrs = [ "active", "description", "private", "archive", "file_quota", "default_gateway_caps", "allow_anon", "closure" ] write_attrs_api_required = write_attrs def owned_by(self, user): return user.owner_id == self.owner_id def need_gateway_auth(self): """ Do we require an authentic gateway to interact with us? (i.e. do we forbid anonymous users)? """ if not self.allow_anon: return True return False def protobuf(self, volume_metadata, **kwargs): """ Convert to a protobuf (ms_volume_metadata) """ volume_metadata.owner_id = kwargs.get('owner_id', self.owner_id) volume_metadata.blocksize = kwargs.get('blocksize', self.blocksize) volume_metadata.volume_id = kwargs.get('volume_id', self.volume_id) volume_metadata.name = kwargs.get('name', self.name) volume_metadata.description = kwargs.get('description', self.description) volume_metadata.volume_version = kwargs.get('volume_version', self.version) volume_metadata.cert_version = kwargs.get('cert_version', self.cert_version) volume_metadata.volume_public_key = kwargs.get( 'metadata_public_key', self.metadata_public_key) volume_metadata.archive = kwargs.get('archive', self.archive) volume_metadata.private = kwargs.get('private', self.private) volume_metadata.allow_anon = kwargs.get('allow_anon', self.allow_anon) if kwargs.get('closure', self.closure) is not None: volume_metadata.cache_closure_text = kwargs.get( 'closure', self.closure) # sign it volume_metadata.signature = "" data = volume_metadata.SerializeToString() sig = self.sign_message(data) volume_metadata.signature = sig return def protobuf_gateway_cert(self, gateway_cert, gateway, sign=True, need_closure=True): """ Given an ms_gateway_cert protobuf and a gateway record, have the gateway populate the cert protobuf and then have the Volume optionally sign it with its private key. """ gateway.protobuf_cert(gateway_cert, need_closure=need_closure) gateway_cert.signature = "" if sign: # sign the cert data = gateway_cert.SerializeToString() sig = self.sign_message(data) gateway_cert.signature = sig return def protobuf_gateway_cert_manifest_record(self, cert_block, g_id, gateway_type, caps, cert_version): """ Populate a protobuf manifest entry with a gateway's certificate information. """ cert_block.gateway_id = g_id cert_block.start_id = gateway_type cert_block.end_id = caps cert_block.block_versions.append(cert_version) def protobuf_gateway_cert_manifest(self, manifest, include_cert=None, sign=True): """ Generate a specially-crafted manifest protobuf, which a gateway can use to learn the IDs and types of all gateways in the Volume, as well as their certs' versions. """ manifest.volume_id = self.volume_id manifest.coordinator_id = 0 manifest.file_id = 0 manifest.owner_id = 0 manifest.file_version = self.cert_version manifest.mtime_sec = 0 manifest.mtime_nsec = 0 manifest.fent_mtime_sec = 0 manifest.fent_mtime_nsec = 0 sz = 0 # query certificate versions, types, and caps of all gateways that need to be trusted listing = Gateway.ListAll( { "Gateway.volume_id ==": self.volume_id, "Gateway.need_cert ==": True }, projection=["g_id", "gateway_type", "cert_version", "caps"]) # if the caller wants to include a particular gateway's cert, do so has_included_cert = False for gateway_metadata in listing: cert_block = manifest.block_url_set.add() self.protobuf_gateway_cert_manifest_record( cert_block, gateway_metadata.g_id, gateway_metadata.gateway_type, gateway_metadata.caps, gateway_metadata.cert_version) logging.info( "cert block: (%s, %s, %s, %x)" % (gateway_metadata.gateway_type, gateway_metadata.g_id, gateway_metadata.cert_version, gateway_metadata.caps)) sz += 1 if gateway_metadata.g_id == include_cert: has_included_cert = True if not has_included_cert and include_cert is not None: # get this gateway's cert as well gw = Gateway.Read(include_cert) if gw is not None: cert_block = manifest.block_url_set.add() self.protobuf_gateway_cert_manifest_record( cert_block, gw.g_id, gw.gateway_type, gw.caps, gw.cert_version) logging.info("cert block (included for %s): (%s, %s, %s, %x)" % (include_cert, gw.gateway_type, gw.g_id, gw.cert_version, gw.caps)) sz += 1 manifest.size = sz manifest.signature = "" if sign: data = manifest.SerializeToString() sig = self.sign_message(data) manifest.signature = sig return def is_gateway_in_volume(self, gateway): """ Determine whether a given Gateway instance belongs to this Volume. If the Volume is not private, then it "belongs" by default. """ if self.allow_anon: return True return gateway.volume_id == self.volume_id def sign_message(self, data): """ Return the base64-encoded crypto signature of the data, signed with our metadata private key. """ signature = Volume.auth_sign(self.metadata_private_key, data) if signature is None: raise Exception("Failed to sign data") sigb64 = base64.b64encode(signature) return sigb64 @classmethod def Create(cls, user, **kwargs): """ Given volume data, store it. Update the corresponding SyndicateUser atomically along with creating the Volume so that the SyndicateUser owns the Volume. Arguments: user -- SyndicateUser instance that will own this Volume Required keyword arguments: name -- name of the Volume (str) blocksize -- size of the Volume's blocks in bytes (int) description -- description of the Volume (str) private -- whether or not this Volume is visible to other users (bool) Optional keyword arguments: metadata_private_key -- PEM-encoded RSA private key, 4096 bits (str) archive -- whether or not this Volume is populated only by Acquisition Gateways (bool) default_gateway_caps -- bitfield of capabilities Gateways created within this Volume should receive """ # sanity check if not user: raise Exception("No user given") kwargs[ 'owner_id'] = 0 # will look up user and fill with owner ID once we validate input. Volume.fill_defaults(kwargs) # extract public key from private key if needed Volume.extract_keys('metadata_public_key', 'metadata_private_key', kwargs, VOLUME_RSA_KEYSIZE) # Validate missing = Volume.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) invalid = Volume.validate_fields(kwargs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) # vet the keys for key_field in ['metadata_public_key', 'metadata_private_key']: key_str = kwargs[key_field] valid = cls.is_valid_key(key_str, VOLUME_RSA_KEYSIZE) if not valid: raise Exception("Key must be a %s-bit RSA key" % (VOLUME_RSA_KEYSIZE)) # attempt to create the Volume volume_id = random.randint(1, 2**63 - 1) volume_key_name = Volume.make_key_name(volume_id=volume_id) volume_key = storagetypes.make_key(Volume, volume_key_name) # put the Volume and nameholder at the same time---there's a good chance we'll succeed volume_nameholder_fut = VolumeNameHolder.create_async( kwargs['name'], volume_id) volume_fut = Volume.get_or_insert_async( volume_key_name, name=kwargs['name'], blocksize=kwargs['blocksize'], description=kwargs['description'], owner_id=user.owner_id, volume_id=volume_id, active=kwargs.get('active', False), version=1, cert_version=1, private=kwargs['private'], archive=kwargs['archive'], allow_anon=kwargs['allow_anon'], metadata_public_key=kwargs['metadata_public_key'], metadata_private_key=kwargs['metadata_private_key'], default_gateway_caps=kwargs['default_gateway_caps']) storagetypes.wait_futures([volume_nameholder_fut, volume_fut]) # verify that there was no collision volume = volume_fut.get_result() volume_nameholder = volume_nameholder_fut.get_result() if volume_nameholder.volume_id != volume_id: # name collision storagetypes.deferred.defer(Volume.delete_all, [volume_key]) raise Exception("Volume '%s' already exists!" % kwargs['name']) if volume.volume_id != volume_id: # ID collision storagetypes.deferred.defer(Volume.delete_all, [volume_key, volume_nameholder.key]) raise Exception("Volume ID collision. Please try again") # set permissions req = VolumeAccessRequest.create_async( user.owner_id, volume_id, kwargs['name'], random.randint(-2**63, 2**63 - 1), VolumeAccessRequest.STATUS_GRANTED, gateway_caps=kwargs['default_gateway_caps'], allowed_gateways=(1 << GATEWAY_TYPE_AG) | (1 << GATEWAY_TYPE_UG) | (1 << GATEWAY_TYPE_RG), request_message="Created").get_result() return volume_key @classmethod def Read(cls, volume_name_or_id, async=False, use_memcache=True):
class GatewayDriver(storagetypes.Object): """ Gateway driver, addressed by hash. """ driver_hash = storagetypes.String() # hex string driver_text = storagetypes.Blob() refcount = storagetypes.Integer() @classmethod def hash_driver(cls, driver_text): h = HashAlg.new() h.update(driver_text) return h.hexdigest() @classmethod def make_key_name(cls, driver_hash): return "GatewayDriver: hash=%s" % (driver_hash) @classmethod def create_or_ref(cls, _text): """ Create a new driver, or re-ref the existing one. Do so atomically. """ driver_hash = cls.hash_driver(_text) def txn(): dk = storagetypes.make_key( GatewayDriver, GatewayDriver.make_key_name(driver_hash)) d = dk.get() f = None if d is None: d = GatewayDriver(key=dk, driver_hash=driver_hash, driver_text=_text, refcount=1) d.put() else: d.refcount += 1 d.put() return d return storagetypes.transaction(txn) @classmethod def ref(cls, driver_hash): """ Increment reference count. Do this in an "outer" transaction (i.e. Gateway.Update) """ dk = storagetypes.make_key(GatewayDriver, cls.make_key_name(driver_hash)) d = dk.get() if d is None: return False d.refcount += 1 d.put() return True @classmethod def unref(cls, driver_hash): """ Unref a driver Delete it if its ref count goes non-positive. Do this in an "outer" transaction (i.e. Gateway.Delete, Gateway.Update) """ dk = storagetypes.make_key(GatewayDriver, cls.make_key_name(driver_hash)) d = dk.get() if d is None: return True d.refcount -= 1 if d.refcount <= 0: dk.delete() else: d.put() return True @classmethod def unref_async(cls, driver_hash): """ Unref a driver, asynchronously Delete it if its ref count goes non-positive. Do this in an "outer" transaction (i.e. Gateway.Delete, Gateway.Update) """ dk = storagetypes.make_key(GatewayDriver, cls.make_key_name(driver_hash)) d = dk.get() if d is None: return True d.refcount -= 1 if d.refcount <= 0: dk.delete_async() else: d.put_async() return True
class Gateway(storagetypes.Object): # signed gateaway certificate from the user gateway_cert = storagetypes.Blob( ) # protobuf'ed gateway certificate generated and signed by the gateway owner upon creation # all of the below information is derived from the above signed gateway certificate. # it is NOT filled in by any method. gateway_type = storagetypes.Integer(default=0) owner_id = storagetypes.Integer( default=-1) # ID of the SyndicateUser that owns this gateway host = storagetypes.String() port = storagetypes.Integer() name = storagetypes.String() # name of this gateway g_id = storagetypes.Integer() volume_id = storagetypes.Integer(default=-1) deleted = storagetypes.Boolean(default=False) gateway_public_key = storagetypes.Text( ) # PEM-encoded RSA public key to verify control-plane messages (metadata) sent from this gateway. caps = storagetypes.Integer(default=0) # capabilities cert_expires = storagetypes.Integer(default=-1) # -1 means "never expires" cert_version = storagetypes.Integer( default=1) # certificate-related version of this gateway driver_hash = storagetypes.String( ) # driver hash for this gateway (addresses GatewayDriver). hex string, not byte string need_cert = storagetypes.Boolean( default=False ) # whether or not other gateways in the volume need this gateway's certificate (i.e. will this gateway ever serve data) # for RPC key_type = "gateway" required_attrs = ["gateway_cert"] read_attrs_api_required = [ "driver_hash", "host", "port", "owner_id", "g_id", "gateway_type", "volume_id", "cert_version", "cert_expires", "caps", ] read_attrs = [ "gateway_public_key", "name", ] + read_attrs_api_required # fields an API call can set write_attrs = ["gateway_cert"] # attrs from the cert that are allowed to change between cert versions modifiable_cert_attrs = [ "gateway_type", "host", "port", "caps", "cert_expires", "cert_version", "driver_hash", "gateway_public_key" ] write_attrs_api_required = write_attrs default_values = {"gateway_cert": ""} key_attrs = ["g_id"] validators = { "name": (lambda cls, value: len( unicode(value).translate(dict((ord(char), None) for char in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.: ")) ) == 0 \ and not is_int(value) \ and len(value) > 0 ), "gateway_public_key": (lambda cls, value: Gateway.is_valid_key( value, GATEWAY_RSA_KEYSIZE ) and Gateway.is_public_key( value ) ) } @classmethod def needs_cert(cls, gateway_type, caps): """ Given a gateway's capabilities, will another gateway need its certificate? """ if (caps & (GATEWAY_CAP_WRITE_METADATA | GATEWAY_CAP_WRITE_DATA | GATEWAY_CAP_COORDINATE)) != 0: return True return False def owned_by(self, user): return user.owner_id == self.owner_id or self.owner_id == GATEWAY_ID_ANON def load_pubkey(self, pubkey_str, in_base64=True): """ Load a PEM-encoded RSA public key. if in_base64 == True, then try to base64-decode it first (i.e. the PEM-encoded public key is itself base64-encoded again) return 0 on success return -EINVAL if the key is invalid return -EEXIST if the key is the same as the one we have in this Gateway """ pubkey_str_unencoded = None if in_base64: pubkey_str_unencoded = base64.b64decode(pubkey_str) else: pubkey_str_unencoded = pubkey_str if not Gateway.is_valid_key(pubkey_str_unencoded, GATEWAY_RSA_KEYSIZE): return -errno.EINVAL new_public_key = CryptoKey.importKey(pubkey_str_unencoded).exportKey() if self.gateway_public_key is not None and new_public_key == self.gateway_public_key: return -errno.EEXIST self.gateway_public_key = new_public_key return 0 def protobuf_cert(self, cert_pb): """ Populate an ms_gateway_cert structure from our cert """ gateway_cert_pb = ms_pb2.ms_gateway_cert.ParseFromString( self.gateway_cert) cert_pb.CopyFrom(gateway_cert_pb) def check_caps(self, caps): """ Given a bitmask of caps, verify that all of them are met by our caps. """ return (self.caps & caps) == caps def verify_message(self, msg): """ Verify the authenticity of a received message with a signature field (which should store a base64-encoded signature) """ sig = msg.signature sig_bin = base64.b64decode(sig) msg.signature = "" msg_str = msg.SerializeToString() ret = self.auth_verify(self.gateway_public_key, msg_str, sig_bin) msg.signature = sig return ret def authenticate_session(self, g_type, g_id, url, signature_b64): """ Verify that the signature over the constructed string "${g_type}_${g_id}:${url}" was signed by this gateway's private key. """ sig = base64.b64decode(signature_b64) data = "%s_%s:%s" % (g_type, g_id, url) ret = self.auth_verify(self.gateway_public_key, data, sig) return ret @classmethod def cert_to_dict(cls, gateway_cert): """ Convert a protobuf structure to a dict of values, using the Gateway property names. """ # unpack certificate cert_version = gateway_cert.version gateway_name = gateway_cert.name gateway_type = gateway_cert.gateway_type gateway_id = gateway_cert.gateway_id host = gateway_cert.host port = gateway_cert.port pubkey_pem = gateway_cert.public_key cert_expires = gateway_cert.cert_expires requested_caps = gateway_cert.caps driver_hash = binascii.hexlify(gateway_cert.driver_hash) volume_id = gateway_cert.volume_id owner_id = gateway_cert.owner_id kwargs = { "cert_version": cert_version, "name": gateway_name, "gateway_type": gateway_type, "host": host, "port": port, "gateway_public_key": pubkey_pem, "cert_expires": cert_expires, "caps": requested_caps, "driver_hash": driver_hash, "volume_id": volume_id, "owner_id": owner_id, "g_id": gateway_id, "gateway_cert": gateway_cert.SerializeToString() } return kwargs @classmethod def Create(cls, user, volume, gateway_cert, driver_text): """ Create a gateway, using its user-signed gateway certificate. NOTE: the caller must verify the authenticity of the certificate. """ kwargs = cls.cert_to_dict(gateway_cert) # sanity check if kwargs['volume_id'] != volume.volume_id: raise Exception("Volume ID mismatch: cert has %s; expected %s" % (kwargs['volume_id'], volume.volume_id)) if kwargs['owner_id'] != user.owner_id: # this is only okay if the user is the volume owner, and the gateway ID is the anonymous gateway if not (kwargs['owner_id'] == USER_ID_ANON and volume.owner_id == user.owner_id): raise Exception("User ID mismatch: cert has %s; expected %s" % (kwargs['owner_id'], user.owner_id)) # sanity check: do we have everything we need? missing = cls.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) # sanity check: are our fields valid? invalid = cls.validate_fields(kwargs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) # sanity check: does the driver match the driver's hash in the cert? if driver_text is not None: driver_hash = GatewayDriver.hash_driver(driver_text) if driver_hash != binascii.hexlify(gateway_cert.driver_hash): raise Exception( "Driver hash mismatch: len = %s, expected = %s, got = %s" % (len(driver_text), driver_hash, binascii.hexlify(cert.driver_hash))) gateway_type = kwargs['gateway_type'] # enforce cert distribution kwargs['need_cert'] = Gateway.needs_cert(gateway_type, kwargs['caps']) g_id = kwargs['g_id'] g_key_name = Gateway.make_key_name(g_id=g_id) g_key = storagetypes.make_key(cls, g_key_name) # create a nameholder and this gateway at once---there's a good chance we'll succeed futs = [] gateway_nameholder_fut = GatewayNameHolder.create_async( kwargs['name'], g_id) gateway_fut = cls.get_or_insert_async(g_key_name, **kwargs) futs = [gateway_nameholder_fut, gateway_fut] gateway_driver = None if driver_text is not None: gateway_driver = GatewayDriver.create_or_ref(driver_text) # wait for operations to complete storagetypes.wait_futures(futs) # check for collision... gateway_nameholder = gateway_nameholder_fut.get_result() gateway = gateway_fut.get_result() to_rollback = [] if gateway_driver is not None: to_rollback.append(gateway_driver.key) if gateway_nameholder.g_id != g_id: # name collision... to_rollback.append(g_key) storagetypes.deferred.defer(Gateway.delete_all, to_rollback) raise Exception("Gateway '%s' already exists!" % kwargs['name']) if gateway.g_id != g_id: # ID collision... to_rollback.append(gateway_nameholder.key) to_rollback.append(g_key) storagetypes.deferred.defer(Gateway.delete_all, to_rollback) raise Exception("Gateway ID collision. Please try again.") # we're good! return g_key @classmethod @storagetypes.concurrent def Read_Async(cls, key, deleted=False): gw = yield key.get_async() if gw is None: storagetypes.concurrent_return(None) if gw.deleted and not deleted: storagetypes.concurrent_return(None) storagetypes.concurrent_return(gw) @classmethod def Read(cls, g_name_or_id, async=False, use_memcache=True, deleted=False):
class Gateway(storagetypes.Object): gateway_type = storagetypes.Integer(default=0) owner_id = storagetypes.Integer( default=-1) # ID of the SyndicateUser that owns this gateway host = storagetypes.String() port = storagetypes.Integer() name = storagetypes.String() # name of this gateway g_id = storagetypes.Integer() volume_id = storagetypes.Integer(default=-1) gateway_public_key = storagetypes.Text( ) # PEM-encoded RSA public key to verify control-plane messages (metadata) sent from this gateway. encrypted_gateway_private_key = storagetypes.Text( ) # optional: corresponding RSA private key, sealed with user's password. Can only be set on creation. caps = storagetypes.Integer(default=0) # capabilities session_password_hash = storagetypes.Text() session_password_salt = storagetypes.Text() session_timeout = storagetypes.Integer(default=-1, indexed=False) session_expires = storagetypes.Integer( default=-1) # -1 means "never expires" cert_expires = storagetypes.Integer(default=-1) # -1 means "never expires" cert_version = storagetypes.Integer( default=1) # certificate-related version of this gateway closure = storagetypes.Text() # closure data for this gateway need_cert = storagetypes.Boolean( default=False ) # whether or not other gateways in the volume need this gateway's certificate (i.e. will this gateway ever serve data) # for RPC key_type = "gateway" required_attrs = [ "owner_id", "host", "port", "name", "gateway_type", "caps" ] read_attrs_api_required = [ "closure", "host", "port", "owner_id", "g_id", "gateway_type", "volume_id", "session_timeout", "session_expires", "cert_version", "cert_expires", "caps", "encrypted_gateway_private_key" ] read_attrs = ["gateway_public_key", "name"] + read_attrs_api_required write_attrs = [ "closure", "host", "port", "cert_expires", "session_expires", "session_timeout" ] write_attrs_api_required = write_attrs # TODO: session expires in 3600 seconds # TODO: cert expires in 86400 seconds default_values = { "session_expires": (lambda cls, attrs: -1), "cert_version": (lambda cls, attrs: 1), "cert_expires": (lambda cls, attrs: -1), "caps": (lambda cls, attrs: 0), "encrypted_gateway_private_key": (lambda cls, attrs: None) } key_attrs = ["g_id"] validators = { "session_password_hash": (lambda cls, value: len( unicode(value).translate( dict((ord(char), None) for char in "0123456789abcdef"))) == 0), "name": (lambda cls, value: len( unicode(value).translate( dict( (ord(char), None) for char in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.: " ))) == 0 and not is_int(value)), "gateway_public_key": (lambda cls, value: Gateway.is_valid_key(value, GATEWAY_RSA_KEYSIZE)) } @classmethod def safe_caps(cls, gateway_type, given_caps): ''' Get this gateway's capability bits, while making sure that AGs and RGs get hardwired capabilities. ''' if gateway_type == GATEWAY_TYPE_AG: # caps are always read and write metadata return (GATEWAY_CAP_READ_METADATA | GATEWAY_CAP_WRITE_METADATA) elif gateway_type == GATEWAY_TYPE_RG: # caps are always 0 return 0 return given_caps @classmethod def needs_cert(cls, gateway_type, caps): """ Given a gateway's capabilities, will another gateway need its certificate? """ if gateway_type == GATEWAY_TYPE_AG: return True if gateway_type == GATEWAY_TYPE_RG: return True if (caps & (GATEWAY_CAP_WRITE_METADATA | GATEWAY_CAP_WRITE_DATA | GATEWAY_CAP_COORDINATE)) != 0: return True return False def owned_by(self, user): return user.owner_id == self.owner_id def authenticate_session(self, password): """ Verify that the session password is correct """ pw_hash = Gateway.generate_password_hash(password, self.session_password_salt) return pw_hash == self.session_password_hash @classmethod def generate_password_hash(cls, pw, salt): ''' Given a password and salt, generate the hash to store. ''' h = HashAlg.new() h.update(salt) h.update(pw) pw_hash = h.hexdigest() return unicode(pw_hash) @classmethod def generate_password(cls, length): ''' Create a random password of a given length ''' password = "".join([ random.choice( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" ) for i in xrange(length) ]) return password @classmethod def generate_session_password(cls): ''' Generate a session password ''' return cls.generate_password(GATEWAY_SESSION_PASSWORD_LENGTH) @classmethod def generate_session_secrets(cls): """ Generate a password, password hash, and salt for this gateway """ password = cls.generate_session_password() salt = cls.generate_password(GATEWAY_SESSION_SALT_LENGTH) pw_hash = Gateway.generate_password_hash(password, salt) return (password, pw_hash, salt) def regenerate_session_password(self): """ Regenerate a session password. The caller should put() the gateway after this call to save the hash and salt. """ password, pw_hash, salt = Gateway.generate_session_secrets() if self.session_timeout > 0: self.session_expires = now + self.session_timeout else: self.session_expires = -1 self.session_password_hash = pw_hash self.session_password_salt = salt return password def load_pubkey(self, pubkey_str, in_base64=True): """ Load a PEM-encoded RSA public key. if in_base64 == True, then try to base64-decode it first (i.e. the PEM-encoded public key is itself base64-encoded again) return 0 on success return -EINVAL if the key is invalid return -EEXIST if the key is the same as the one we have in this Gateway """ pubkey_str_unencoded = None if in_base64: pubkey_str_unencoded = base64.b64decode(pubkey_str) else: pubkey_str_unencoded = pubkey_str if not Gateway.is_valid_key(pubkey_str_unencoded, GATEWAY_RSA_KEYSIZE): return -errno.EINVAL new_public_key = CryptoKey.importKey(pubkey_str_unencoded).exportKey() if self.gateway_public_key is not None and new_public_key == self.gateway_public_key: return -errno.EEXIST self.gateway_public_key = new_public_key return 0 def protobuf_cert(self, cert_pb, need_closure=False): """ Populate an ms_volume_gateway_cred structure """ cert_pb.version = self.cert_version cert_pb.gateway_type = self.gateway_type cert_pb.owner_id = self.owner_id cert_pb.gateway_id = self.g_id cert_pb.name = self.name cert_pb.host = self.host cert_pb.port = self.port cert_pb.caps = self.caps cert_pb.cert_expires = self.cert_expires cert_pb.volume_id = self.volume_id if self.closure is None or not need_closure: cert_pb.closure_text = "" elif self.closure is not None: cert_pb.closure_text = str(self.closure) cert_pb.signature = "" if self.gateway_public_key != None: cert_pb.public_key = self.gateway_public_key else: cert_pb.public_key = "NONE" def check_caps(self, caps): """ Given a bitmask of caps, verify that all of them are met by our caps. """ return (self.caps & caps) == caps def verify_message(self, msg): """ Verify the authenticity of a received message with a signature field (which should store a base64-encoded signature) """ sig = msg.signature sig_bin = base64.b64decode(sig) msg.signature = "" msg_str = msg.SerializeToString() ret = self.auth_verify(self.gateway_public_key, msg_str, sig_bin) msg.signature = sig return ret @classmethod def Create(cls, user, volume, **kwargs): """ Create a gateway. NOTE: careful--caps are required! don't let users call this directly. """ # enforce volume ID kwargs['volume_id'] = volume.volume_id # enforce ownership--make sure the calling user owns this gateway kwargs['owner_id'] = user.owner_id # populate kwargs with default values for missing attrs cls.fill_defaults(kwargs) # sanity check: do we have everything we need? missing = cls.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) # sanity check: are our fields valid? invalid = cls.validate_fields(kwargs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) # what kind of gateway are we? gateway_type = kwargs['gateway_type'] # set capabilities correctly and safely kwargs['caps'] = cls.safe_caps(gateway_type, volume.default_gateway_caps) # enforce cert generation kwargs['need_cert'] = Gateway.needs_cert(gateway_type, kwargs['caps']) # ID... g_id = random.randint(0, 2**63 - 1) kwargs['g_id'] = g_id g_key_name = Gateway.make_key_name(g_id=g_id) g_key = storagetypes.make_key(cls, g_key_name) # create a nameholder and this gateway at once---there's a good chance we'll succeed gateway_nameholder_fut = GatewayNameHolder.create_async( kwargs['name'], g_id) gateway_fut = cls.get_or_insert_async(g_key_name, **kwargs) # wait for operations to complete storagetypes.wait_futures([gateway_nameholder_fut, gateway_fut]) # check for collision... gateway_nameholder = gateway_nameholder_fut.get_result() gateway = gateway_fut.get_result() if gateway_nameholder.g_id != g_id: # name collision... storagetypes.deferred.defer(Gateway.delete_all, [g_key]) raise Exception("Gateway '%s' already exists!" % kwargs['name']) if gateway.g_id != g_id: # ID collision... storagetypes.deferred.defer(Gateway.delete_all, [gateway_nameholder.key, g_key]) raise Exception("Gateway ID collision. Please try again.") # we're good! return g_key @classmethod def Read(cls, g_name_or_id, async=False, use_memcache=True):
class Closure(storagetypes.Object): closure_id = storagetypes.Integer(default=0) # unique ID of this closure name = storagetypes.String(default="") # name of this closure owner_id = storagetypes.Integer(default=0) # owner of this closure public = storagetypes.Boolean( default=False ) # whether or not other users' gateways can access this closure blob_ref = storagetypes.Text() # reference to the closure blob # for RPC key_type = "closure" required_attrs = ["owner_id", "public", "blob_ref"] read_attrs_api_required = ["blob_ref"] read_attrs = ["closure_id", "name", "public"] + read_attrs_api_required write_attrs = ["blob_ref"] write_attrs_api_required = write_attrs key_attrs = ["closure_id"] @classmethod def Create(cls, user, **kwargs): """ Create a closure. Only do this after the closure binary has been uploaded successfully. """ # enforce ownership--make sure the calling user owns this closure kwargs['owner_id'] = user.owner_id # populate kwargs with default values for missing attrs cls.fill_defaults(kwargs) # sanity check: do we have everything we need? missing = cls.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) # sanity check: are our fields valid? invalid = cls.validate_fields(kwargs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) # ID... closure_id = random.randint(0, 2**63 - 1) kwargs['closure_id'] = closure_id closure_key_name = Closure.make_key_name(closure_id=closure_id) closure_key = storagetypes.make_key(cls, closure_key_name) # create a nameholder and this closure at once---there's a good chance we'll succeed closure_nameholder_fut = ClosureNameHolder.create_async( kwargs['name'], closure_id) closure_fut = cls.get_or_insert_async(closure_key_name, **kwargs) # wait for operations to complete storagetypes.wait_futures([closure_nameholder_fut, closure_fut]) # check for collision... closure_nameholder = closure_nameholder_fut.get_result() closure = closure_fut.get_result() if closure_nameholder.closure_id != closure_id: # name collision... storagetypes.deferred.defer(Closure.delete_all, [closure_key]) raise Exception("Closure '%s' already exists!" % kwargs['name']) if closure.closure_id != closure_id: # ID collision... storagetypes.deferred.defer(Closure.delete_all, [closure_nameholder.key, closure_key]) raise Exception("Closure ID collision. Please try again.") # we're good! return closure_key @classmethod def Read(cls, closure_name_or_id, async=False, use_memcache=True):
class MSBaselinePerformanceType(storagetypes.Object): """ Record to get and put once per baseline performance test. """ rec_txt = storagetypes.String(default="")
class SyndicateUser(storagetypes.Object): email = storagetypes.String() # used as the username to Syndicate owner_id = storagetypes.Integer() # numeric ID for gateways admin_id = storagetypes.Integer() # which admin made this user? max_volumes = storagetypes.Integer( default=10 ) # how many Volumes can this user create? (-1 means unlimited) max_gateways = storagetypes.Integer( default=10 ) # how many gateways can this user create? (-1 means unlimited) is_admin = storagetypes.Boolean( default=False, indexed=False) # is this user an administrator? public_key = storagetypes.Text( ) # PEM-encoded public key for authenticating this user, or USER_KEY_UNSET if it is not set, or USER_KEY_UNUSED if it will not be used user_cert_protobuf = storagetypes.Blob( ) # protobuf'ed certificate for this user signature = storagetypes.Blob( ) # signature over the data used to generate this record # for RPC key_type = "user" required_attrs = ["email", "public_key"] key_attrs = ["email"] default_values = { "max_volumes": (lambda cls, attrs: 10), "max_gateways": (lambda cls, attrs: 10), "is_admin": (lambda cls, attrs: False), } validators = { "email": (lambda cls, value: valid_email(value)), "public_key": (lambda cls, value: cls.is_valid_key( value, USER_RSA_KEYSIZE) and cls.is_public_key(value)) } read_attrs_api_required = [ "email", "owner_id", "max_volumes", "max_gateways", "public_key", ] read_attrs = read_attrs_api_required write_attrs_api_required = [ "public_key", ] write_attrs_admin_required = ["max_volumes", "max_gateways", "is_admin"] write_attrs = write_attrs_api_required + write_attrs_admin_required # what fields in the cert can change? modifiable_cert_fields = ["public_key", "max_volumes", "max_gateways"] def owned_by(self, user): return user.owner_id == self.owner_id @classmethod def Authenticate(cls, email, data, data_signature): """ Authenticate a user via public-key cryptography. Verify that data was signed by the user's private key, given the signature and data. (use RSA PSS for security). Return the user on success; False on authentication error; None if the user doesn't exist """ user = SyndicateUser.Read(email) if user == None: return None ret = cls.auth_verify(user.public_key, data, data_signature) if not ret: logging.error("Verification failed for %s" % email) return False else: return user @classmethod def cert_to_dict(cls, user_cert): attrs = { 'email': str(user_cert.email), 'owner_id': user_cert.user_id, 'public_key': str(user_cert.public_key), 'admin_id': user_cert.admin_id, 'max_volumes': user_cert.max_volumes, 'max_gateways': user_cert.max_gateways, 'is_admin': user_cert.is_admin, 'signature': str(user_cert.signature), 'user_cert_protobuf': user_cert.SerializeToString() } return attrs @classmethod def Create(cls, user_cert): """ Create a SyndicateUser from a user_cert. NOTE: the caller will need to have validated the user cert """ kwargs = cls.cert_to_dict(user_cert) email = kwargs['email'] missing = SyndicateUser.find_missing_attrs(kwargs) if len(missing) != 0: raise Exception("Missing attributes: %s" % (", ".join(missing))) invalid = SyndicateUser.validate_fields(kwargs) if len(invalid) != 0: raise Exception("Invalid values for fields: %s" % (", ".join(invalid))) user_key_name = SyndicateUser.make_key_name(email=email) user = storagetypes.memcache.get(user_key_name) if user == None: user_key = storagetypes.make_key(SyndicateUser, user_key_name) user = user_key.get() if user == None: # create! user = SyndicateUser.get_or_insert(user_key_name, **kwargs) # check for collisions if user.owner_id != kwargs['owner_id']: # collision raise Exception("User '%s' already exists" % email) return user.key else: raise Exception("User '%s' already exists" % email) else: raise Exception("User '%s' already exists" % email) @classmethod def CreateAdmin(cls, email, owner_id, public_key, syndicate_private_key): """ Create the admin user. Called when the MS initializes itself for the first time """ import common.api as api admin_cert = ms_pb2.ms_user_cert() admin_cert.user_id = owner_id admin_cert.email = email admin_cert.public_key = public_key admin_cert.admin_id = owner_id admin_cert.max_volumes = -1 admin_cert.max_gateways = -1 admin_cert.is_admin = True admin_cert.signature = "" admin_cert_str = admin_cert.SerializeToString() sig = api.sign_data(syndicate_private_key, admin_cert_str) admin_cert.signature = base64.b64encode(sig) return SyndicateUser.Create(admin_cert) @classmethod def Read(cls, email_or_owner_id, async=False): """ Read a SyndicateUser Arguments: email_or_owner_id -- Email address of the user to read, or the owner ID (str or int) """ owner_id = None email = None try: owner_id = int(email_or_owner_id) except: email = email_or_owner_id if owner_id is not None: return cls.Read_ByOwnerID(owner_id, async=async) user_key_name = SyndicateUser.make_key_name(email=email) user_key = storagetypes.make_key(SyndicateUser, user_key_name) user = storagetypes.memcache.get(user_key_name) if user == None: if async: return user_key.get_async(use_memcache=False) else: user = user_key.get(use_memcache=False) if not user: return None else: storagetypes.memcache.set(user_key_name, user) elif async: user = storagetypes.FutureWrapper(user) return user