def delete_volume_and_friends( cls, volume_id, volume_name ):
"""
Delete the following for a particular volume, as a deferred task:
the Volume
# all Volume access requests
the Volume name holder
Does not delete attached gateways.
"""
futs = []
# delete volume
volume_key = storagetypes.make_key( Volume, Volume.make_key_name( volume_id=volume_id ) )
futs.append( volume_key.delete_async() )
# delete volume nameholder
volume_nameholder_key = storagetypes.make_key( VolumeNameHolder, VolumeNameHolder.make_key_name( volume_name ) )
futs.append( volume_nameholder_key.delete_async() )
# delete volume access requests
#volume_access_requests_fut = VolumeAccessRequest.DeleteAccessRequestsByVolume( volume_id, async=True )
#futs.append( volume_access_requests_fut )
storagetypes.wait_futures( futs )
def Delete(cls, g_name_or_id):
"""
Given a gateway ID, delete the corresponding gateway
"""
gateway = Gateway.Read(g_name_or_id)
if gateway:
g_id = gateway.g_id
else:
raise Exception("No such Gateway '%s'" % g_name_or_id)
key_name = Gateway.make_key_name(g_id=g_id)
g_key = storagetypes.make_key(cls, key_name)
g_name_key = storagetypes.make_key(
GatewayNameHolder, GatewayNameHolder.make_key_name(gateway.name))
g_delete_fut = g_key.delete_async()
g_name_delete_fut = g_name_key.delete_async()
Gateway.FlushCache(g_id)
g_name_to_id_cache_key = Gateway.Read_ByName_name_cache_key(
g_name_or_id)
storagetypes.memcache.delete(g_name_to_id_cache_key)
storagetypes.wait_futures([g_delete_fut, g_name_delete_fut])
return True
def Delete(cls, cls_name_or_id):
"""
Given a closure ID, delete the corresponding closure.
NOTE: Make sure that no gateway references this closure first.
"""
closure = Closure.Read(cls_name_or_id)
if closure is not None:
cls_id = closure.closure_id
else:
raise Exception("No such Closure '%s'" % cls_name_or_id)
key_name = Closure.make_key_name(closure_id=cls_id)
cls_key = storagetypes.make_key(cls, key_name)
cls_name_key = storagetypes.make_key(
ClosureNameHolder, ClosureNameHolder.make_key_name(closure.name))
cls_delete_fut = cls_key.delete_async()
cls_name_delete_fut = cls_name_key.delete_async()
Closure.FlushCache(cls_id)
cls_name_to_id_cache_key = Closure.Read_ByName_name_cache_key(
cls_name_or_id)
storagetypes.memcache.delete(cls_name_to_id_cache_key)
storagetypes.wait_futures([cls_delete_fut, cls_name_delete_fut])
return True
def Delete( cls, cls_name_or_id ):
"""
Given a closure ID, delete the corresponding closure.
NOTE: Make sure that no gateway references this closure first.
"""
closure = Closure.Read( cls_name_or_id )
if closure is not None:
cls_id = closure.closure_id
else:
raise Exception("No such Closure '%s'" % cls_name_or_id )
key_name = Closure.make_key_name( closure_id=cls_id )
cls_key = storagetypes.make_key( cls, key_name )
cls_name_key = storagetypes.make_key( ClosureNameHolder, ClosureNameHolder.make_key_name( closure.name ) )
cls_delete_fut = cls_key.delete_async()
cls_name_delete_fut = cls_name_key.delete_async()
Closure.FlushCache( cls_id )
cls_name_to_id_cache_key = Closure.Read_ByName_name_cache_key( cls_name_or_id )
storagetypes.memcache.delete( cls_name_to_id_cache_key )
storagetypes.wait_futures( [cls_delete_fut, cls_name_delete_fut] )
return True
def Delete( cls, g_name_or_id ):
"""
Given a gateway ID, delete the corresponding gateway.
Unref the driver as well.
"""
gateway = Gateway.Read( g_name_or_id )
if gateway:
g_id = gateway.g_id
else:
raise Exception("No such Gateway '%s'" % g_name_or_id )
key_name = Gateway.make_key_name( g_id=g_id )
g_key = storagetypes.make_key( cls, key_name )
g_name_key = storagetypes.make_key( GatewayNameHolder, GatewayNameHolder.make_key_name( gateway.name ) )
g_delete_fut = g_key.delete_async()
g_name_delete_fut = g_name_key.delete_async()
driver_fut = GatewayDriver.unref_async( gateway.driver_hash )
storagetypes.wait_futures( [g_delete_fut, g_name_delete_fut, driver_fut] )
Gateway.FlushCache( g_id )
Gateway.FlushCacheDriver( gateway.driver_hash )
g_name_to_id_cache_key = Gateway.Read_ByName_name_cache_key( g_name_or_id )
storagetypes.memcache.delete( g_name_to_id_cache_key )
return True
def delete_volume_and_friends(cls, volume_id, volume_name):
"""
Delete the following for a particular volume, as a deferred task:
the Volume
all Volume access requests
the Volume name holder
"""
futs = []
# delete volume
volume_key = storagetypes.make_key(
Volume, Volume.make_key_name(volume_id=volume_id))
futs.append(volume_key.delete_async())
# delete volume nameholder
volume_nameholder_key = storagetypes.make_key(
VolumeNameHolder, VolumeNameHolder.make_key_name(volume_name))
futs.append(volume_nameholder_key.delete_async())
# delete volume access requests
volume_access_requests_fut = VolumeAccessRequest.DeleteAccessRequestsByVolume(
volume_id, async=True)
futs.append(volume_access_requests_fut)
storagetypes.wait_futures(futs)
def __compactify_remove_index_async( cls, volume_id, parent_id, dead_file_id, dead_dir_index ):
"""
Remove a freed index slot's node data.
"""
idx_key_name = MSEntryDirEntIndex.make_key_name( volume_id, parent_id, dead_dir_index )
ent_key_name = MSEntryEntDirIndex.make_key_name( volume_id, dead_file_id )
idx_key = storagetypes.make_key( MSEntryDirEntIndex, idx_key_name )
ent_key = storagetypes.make_key( MSEntryEntDirIndex, ent_key_name )
@storagetypes.concurrent
def delete_index_if_unallocated():
idx_node = yield idx_key.get_async( use_cache=False, use_memcache=False )
if idx_node is None:
# already gone
storagetypes.concurrent_return( 0 )
if not idx_node.alloced:
yield idx_key.delete_async()
storagetypes.concurrent_return( 0 )
yield ent_key.delete_async(), storagetypes.transaction_async( delete_index_if_unallocated )
storagetypes.memcache.delete_multi( [idx_key_name, ent_key_name] )
def put_txn():
volume_cert_bundle = cls.Get( volume_id )
if volume_cert_bundle is not None:
existing_cert = cls.Load( volume_cert_bundle )
if existing_cert.volume_id != volume_id:
raise Exception("BUG: existing cert bundle is for %s, but expected %s" % (volume_id, existing_cert.volume_id))
if existing_cert.file_version > cert.file_version:
raise Exception("Stale volume cert version: expected >= %s, got %s" % (existing_cert.file_version, cert.file_version))
if existing_cert.mtime_sec > cert.mtime_sec or (existing_cert.mtime_sec == cert.mtime_sec and existing_cert.mtime_nsec > cert.mtime_nsec):
# stale
raise Exception("Stale cert bundle timestamp: expected > %s.%s, got %s.%s" % (volume_id, existing_cert.mtime_sec, existing_cert.mtime_nsec, cert.mtime_sec, cert.mtime_nsec))
volume_cert_bundle.cert_protobuf = cert_protobuf
volume_cert_bundle.put()
storagetypes.memcache.delete( VolumeCertBundle.make_key_name( volume_id ) )
else:
volume_cert_bundle = VolumeCertBundle( key=storagetypes.make_key( VolumeCertBundle, VolumeCertBundle.make_key_name( volume_id ) ), volume_id=volume_id, cert_protobuf=cert_protobuf )
volume_cert_bundle.put()
return True
def __update_index_node_async( cls, volume_id, parent_id, file_id, dir_index, alloced, **attrs ):
"""
Set the allocation status of a directory index node (but not its matching entry index node).
Return 0 on success
Return -EINVAL if the given file_id doesn't match the directory index node's file_id
Return -EEXIST if the given directory index node's allocation status is the same as alloced
"""
index_key_name = MSEntryDirEntIndex.make_key_name( volume_id, parent_id, dir_index )
index_key = storagetypes.make_key( MSEntryDirEntIndex, index_key_name )
old_alloced = None
idx = yield index_key.get_async()
if idx is None:
old_alloced = alloced
idx = MSEntryDirEntIndex( key=index_key, volume_id=volume_id, parent_id=parent_id, file_id=file_id, dir_index=dir_index, alloced=alloced, **attrs )
else:
if idx.file_id != file_id:
# wrong node
storagetypes.concurrent_return( -errno.EINVAL )
old_alloced = idx.alloced
if old_alloced != alloced:
# changing allocation status
idx.populate( -1, volume_id=volume_id, parent_id=parent_id, file_id=file_id, dir_index=dir_index, alloced=alloced, **attrs )
yield idx.put_async()
storagetypes.concurrent_return( 0 )
else:
storagetypes.concurrent_return( -errno.EEXIST )
def Delete( cls, email ):
'''
Delete a SyndicateUser
Arguments:
email -- Email of the user to delete (str)
'''
# can't delete the original admin account
if email == ADMIN_EMAIL:
raise Exception("Cannot delete Syndicate owner")
user_key_name = SyndicateUser.make_key_name( email=email)
user_key = storagetypes.make_key( SyndicateUser, user_key_name )
def delete_func( user_key ):
user = user_key.get()
if user == None:
# done!
return True
user_key.delete()
return
user_key.delete()
storagetypes.memcache.delete( user_key_name )
return True
def Delete(cls, email):
'''
Delete a SyndicateUser
Arguments:
email -- Email of the user to delete (str)
'''
# can't delete the original admin account
if email == ADMIN_EMAIL:
raise Exception("Cannot delete Syndicate owner")
user_key_name = SyndicateUser.make_key_name(email=email)
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
def delete_func(user_key):
user = user_key.get()
if user == None:
# done!
return True
user_key.delete()
return
user_key.delete()
storagetypes.memcache.delete(user_key_name)
return True
def __update_or_alloc_async( cls, volume_id, parent_id, file_id, dir_index, generation, alloced ):
"""
Update or allocate the index node pair and/or set the directory index node's allocation status, asynchronously.
If the directory index node does not exist, it and its entry index node will be created and the allocation status set accordingly.
If the directory index node exists, but has a different allocation status, then the allocation status will be set accordingly.
If we succeed in allocating a new index node, incremenet the number of children in the parent directory.
Return True on success.
Return False if the index node existed, but the file_id did not match its record or the allocation status did not change.
"""
index_key_name = MSEntryDirEntIndex.make_key_name( volume_id, parent_id, dir_index )
nonce = random.randint( -2**63, 2**63 - 1 )
result = True
idx = yield MSEntryDirEntIndex.get_or_insert_async( index_key_name, volume_id=volume_id, parent_id=parent_id, file_id=file_id, dir_index=dir_index, generation=generation, alloced=alloced, nonce=nonce )
if idx.nonce == nonce:
# created.
if alloced:
logging.info("Directory /%s/%s: allocated index slot for /%s/%s at %s" % (volume_id, parent_id, volume_id, file_id, dir_index))
else:
logging.info("Directory /%s/%s: freed index slot at %s" % (volume_id, parent_id, dir_index))
# need to create an entry index node as well.
entry_key_name = MSEntryEntDirIndex.make_key_name( volume_id, file_id )
entry_key = storagetypes.make_key( MSEntryEntDirIndex, entry_key_name )
entry_idx = MSEntryEntDirIndex( key=entry_key, volume_id=volume_id, parent_id=parent_id, file_id=file_id, dir_index=dir_index, generation=generation, alloced=alloced, nonce=nonce )
yield entry_idx.put_async()
else:
# already exists. changing allocation status?
if idx.alloced != alloced:
# allocation status needs to be changed
# want to change allocation status
rc = yield storagetypes.transaction_async( lambda: cls.__update_index_node_async( volume_id, parent_id, file_id, dir_index, alloced, generation=generation ), xg=True )
if rc == 0:
result = True
else:
logging.error("__update_index_node_async(/%s/%s file_id=%s dir_index=%s alloced=%s) rc = %s" % (volume_id, parent_id, file_id, dir_index, alloced, rc ))
result = False
else:
if alloced and idx.file_id != file_id:
# collision on insertion
logging.error("Directory /%s/%s: collision inserting /%s/%s at %s (occupied by /%s/%s)" % (volume_id, parent_id, volume_id, file_id, dir_index, volume_id, idx.file_id))
result = False
else:
# created/set correctly
result = True
storagetypes.concurrent_return( result )
def update_shard_count( cls, volume_id, num_shards, **txn_args ):
"""
Update the shard count of the volume, but in a transaction.
"""
volume_key = storagetypes.make_key( Volume, Volume.make_key_name( volume_id=volume_id ) )
num_shards = storagetypes.transaction( lambda: Volume.__update_shard_count( volume_key, num_shards ), **txn_args )
return num_shards
def get( self ):
test_key = storagetypes.make_key( MSBaselinePerformanceType, "test_record" )
rec = MSBaselinePerformanceType( key=test_key, rec_txt=self.test_record )
rec.put( use_memcache=False, use_cache=False, use_datastore=True )
out_rec = test_key.get( use_memcache=False, use_cache=False, use_datastore=True )
response_end( self, 200, "OK", "text/plain" )
return
def CreateAdmin(cls, email, openid_url, signing_public_key,
activate_password):
"""
Create the Admin user. NOTE: this will be called repeatedly, so use memcache
"""
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# admin does not exist
attrs = {}
logging.info("Generating admin '%s'" % email)
# fill defaults
SyndicateUser.fill_defaults(attrs)
attrs['email'] = email
attrs['openid_url'] = openid_url
attrs['owner_id'] = random.randint(1, 2**63 - 1)
attrs['is_admin'] = True
# generate password hash and salt
import common.api as api
pw_salt = api.password_salt()
pw_hash = api.hash_password(activate_password, pw_salt)
attrs['activate_password_hash'] = pw_hash
attrs['activate_password_salt'] = pw_salt
# possible that we haven't set the public key yet
if not signing_public_key or len(signing_public_key) == 0:
signing_public_key = cls.USER_KEY_UNSET
attrs['signing_public_key'] = signing_public_key
invalid = SyndicateUser.validate_fields(attrs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user = SyndicateUser.get_or_insert(user_key_name, **attrs)
# check for collisions
if user.owner_id != attrs['owner_id']:
# collision
logging.warning("Admin '%s' already exists" % email)
storagetypes.memcache.set(user_key_name, user)
return user.key
def set_deleted():
# atomically set the gateway to deleted
g_key = storagetypes.make_key( cls, key_name )
gw = g_key.get()
if gw is None:
return None
gw.deleted = True
gw.put()
return gw.key
def set_deleted():
# atomically set the gateway to deleted
g_key = storagetypes.make_key(cls, key_name)
gw = g_key.get()
if gw is None:
return None
gw.deleted = True
gw.put()
return gw.key
def Create(cls, user, **kwargs):
"""
Create a closure.
Only do this after the closure binary has been uploaded successfully.
"""
# enforce ownership--make sure the calling user owns this closure
kwargs['owner_id'] = user.owner_id
# populate kwargs with default values for missing attrs
cls.fill_defaults(kwargs)
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
# sanity check: are our fields valid?
invalid = cls.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
# ID...
closure_id = random.randint(0, 2**63 - 1)
kwargs['closure_id'] = closure_id
closure_key_name = Closure.make_key_name(closure_id=closure_id)
closure_key = storagetypes.make_key(cls, closure_key_name)
# create a nameholder and this closure at once---there's a good chance we'll succeed
closure_nameholder_fut = ClosureNameHolder.create_async(
kwargs['name'], closure_id)
closure_fut = cls.get_or_insert_async(closure_key_name, **kwargs)
# wait for operations to complete
storagetypes.wait_futures([closure_nameholder_fut, closure_fut])
# check for collision...
closure_nameholder = closure_nameholder_fut.get_result()
closure = closure_fut.get_result()
if closure_nameholder.closure_id != closure_id:
# name collision...
storagetypes.deferred.defer(Closure.delete_all, [closure_key])
raise Exception("Closure '%s' already exists!" % kwargs['name'])
if closure.closure_id != closure_id:
# ID collision...
storagetypes.deferred.defer(Closure.delete_all,
[closure_nameholder.key, closure_key])
raise Exception("Closure ID collision. Please try again.")
# we're good!
return closure_key
def Delete( cls, email ):
"""
Delete a SyndicateUser
"""
user_key_name = SyndicateUser.make_key_name( email=email)
user_key = storagetypes.make_key( SyndicateUser, user_key_name )
user_key.delete()
storagetypes.memcache.delete( user_key_name )
return True
def get(self):
test_key = storagetypes.make_key(MSBaselinePerformanceType,
"test_record")
rec = MSBaselinePerformanceType(key=test_key, rec_txt=self.test_record)
rec.put(use_memcache=False, use_cache=False, use_datastore=True)
out_rec = test_key.get(use_memcache=False,
use_cache=False,
use_datastore=True)
response_end(self, 200, "OK", "text/plain")
return
def CreateAdmin( cls, email, openid_url, signing_public_key, activate_password ):
"""
Create the Admin user. NOTE: this will be called repeatedly, so use memcache
"""
user_key_name = SyndicateUser.make_key_name( email=email )
user = storagetypes.memcache.get( user_key_name )
if user == None:
user_key = storagetypes.make_key( SyndicateUser, user_key_name )
user = user_key.get()
if user == None:
# admin does not exist
attrs = {}
logging.info("Generating admin '%s'" % email)
# fill defaults
SyndicateUser.fill_defaults( attrs )
attrs['email'] = email
attrs['openid_url'] = openid_url
attrs['owner_id'] = random.randint( 1, 2**63 - 1 )
attrs['is_admin'] = True
# generate password hash and salt
import common.api as api
pw_salt = api.password_salt()
pw_hash = api.hash_password( activate_password, pw_salt )
attrs['activate_password_hash'] = pw_hash
attrs['activate_password_salt'] = pw_salt
# possible that we haven't set the public key yet
if not signing_public_key or len(signing_public_key) == 0:
signing_public_key = cls.USER_KEY_UNSET
attrs['signing_public_key'] = signing_public_key
invalid = SyndicateUser.validate_fields( attrs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )))
user = SyndicateUser.get_or_insert( user_key_name, **attrs )
# check for collisions
if user.owner_id != attrs['owner_id']:
# collision
logging.warning("Admin '%s' already exists" % email)
storagetypes.memcache.set( user_key_name, user )
return user.key
def RemoveAccessRequest( cls, owner_id, volume_id ):
"""
Delete an access request.
"""
req_key_name = VolumeAccessRequest.make_key_name( owner_id, volume_id )
req_key = storagetypes.make_key( VolumeAccessRequest, req_key_name )
storagetypes.deferred.defer( cls.delete_all, [req_key] )
storagetypes.memcache.delete( req_key_name )
return True
def update_shard_count(cls, volume_id, num_shards, **txn_args):
"""
Update the shard count of the volume, but in a transaction.
"""
volume_key = storagetypes.make_key(
Volume, Volume.make_key_name(volume_id=volume_id))
num_shards = storagetypes.transaction(
lambda: Volume.__update_shard_count(volume_key, num_shards),
**txn_args)
return num_shards
def RemoveAccessRequest(cls, owner_id, volume_id):
"""
Delete an access request.
"""
req_key_name = VolumeAccessRequest.make_key_name(owner_id, volume_id)
req_key = storagetypes.make_key(VolumeAccessRequest, req_key_name)
storagetypes.deferred.defer(cls.delete_all, [req_key])
storagetypes.memcache.delete(req_key_name)
return True
def Delete( cls, volume_id ):
"""
Delete a cert bundle
"""
key_name = VolumeCertBundle.make_key_name( volume_id )
volume_cert_bundle_key = storagetypes.make_key( VolumeCertBundle, key_name )
volume_cert_bundle_key.delete()
storagetypes.memcache.delete( key_name )
return True
def Delete(cls, email):
"""
Delete a SyndicateUser
"""
user_key_name = SyndicateUser.make_key_name(email=email)
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user_key.delete()
storagetypes.memcache.delete(user_key_name)
return True
def Create( cls, user, **kwargs ):
"""
Create a closure.
Only do this after the closure binary has been uploaded successfully.
"""
# enforce ownership--make sure the calling user owns this closure
kwargs['owner_id'] = user.owner_id
# populate kwargs with default values for missing attrs
cls.fill_defaults( kwargs )
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs( kwargs )
if len(missing) != 0:
raise Exception( "Missing attributes: %s" % (", ".join( missing )))
# sanity check: are our fields valid?
invalid = cls.validate_fields( kwargs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )) )
# ID...
closure_id = random.randint( 0, 2**63 - 1 )
kwargs['closure_id'] = closure_id
closure_key_name = Closure.make_key_name( closure_id=closure_id )
closure_key = storagetypes.make_key( cls, closure_key_name )
# create a nameholder and this closure at once---there's a good chance we'll succeed
closure_nameholder_fut = ClosureNameHolder.create_async( kwargs['name'], closure_id )
closure_fut = cls.get_or_insert_async( closure_key_name, **kwargs )
# wait for operations to complete
storagetypes.wait_futures( [closure_nameholder_fut, closure_fut] )
# check for collision...
closure_nameholder = closure_nameholder_fut.get_result()
closure = closure_fut.get_result()
if closure_nameholder.closure_id != closure_id:
# name collision...
storagetypes.deferred.defer( Closure.delete_all, [closure_key] )
raise Exception( "Closure '%s' already exists!" % kwargs['name'] )
if closure.closure_id != closure_id:
# ID collision...
storagetypes.deferred.defer( Closure.delete_all, [closure_nameholder.key, closure_key] )
raise Exception( "Closure ID collision. Please try again." )
# we're good!
return closure_key
def ref( cls, driver_hash ):
"""
Increment reference count.
Do this in an "outer" transaction (i.e. Gateway.Update)
"""
dk = storagetypes.make_key( GatewayDriver, cls.make_key_name( driver_hash ) )
d = dk.get()
if d is None:
return False
d.refcount += 1
d.put()
return True
def txn():
dk = storagetypes.make_key( GatewayDriver, GatewayDriver.make_key_name( driver_hash ) )
d = dk.get()
f = None
if d is None:
d = GatewayDriver( key=dk, driver_hash=driver_hash, driver_text=_text, refcount=1 )
d.put()
else:
d.refcount += 1
d.put()
return d
def ref(cls, driver_hash):
"""
Increment reference count.
Do this in an "outer" transaction (i.e. Gateway.Update)
"""
dk = storagetypes.make_key(GatewayDriver,
cls.make_key_name(driver_hash))
d = dk.get()
if d is None:
return False
d.refcount += 1
d.put()
return True
def GetAccess( cls, owner_id, volume_id ):
"""
Get the access status of a user in a Volume.
"""
req_key_name = VolumeAccessRequest.make_key_name( owner_id, volume_id )
req = storagetypes.memcache.get( req_key_name )
if req != None:
return req
req_key = storagetypes.make_key( VolumeAccessRequest, req_key_name )
req = req_key.get()
if req != None:
storagetypes.memcache.set( req_key_name, req )
return req
def GetAccess(cls, owner_id, volume_id):
"""
Get the access status of a user in a Volume.
"""
req_key_name = VolumeAccessRequest.make_key_name(owner_id, volume_id)
req = storagetypes.memcache.get(req_key_name)
if req != None:
return req
req_key = storagetypes.make_key(VolumeAccessRequest, req_key_name)
req = req_key.get()
if req != None:
storagetypes.memcache.set(req_key_name, req)
return req
def Delete(cls, g_name_or_id):
"""
Given a gateway ID, delete the corresponding gateway.
That is, set it's "deleted" flag so it no longer gets read.
Unref the driver as well.
"""
gateway = Gateway.Read(g_name_or_id)
if gateway:
g_id = gateway.g_id
else:
return True
key_name = Gateway.make_key_name(g_id=g_id)
def set_deleted():
# atomically set the gateway to deleted
g_key = storagetypes.make_key(cls, key_name)
gw = g_key.get()
if gw is None:
return None
gw.deleted = True
gw.put()
return gw.key
storagetypes.transaction(lambda: set_deleted())
g_name_key = storagetypes.make_key(
GatewayNameHolder, GatewayNameHolder.make_key_name(gateway.name))
g_name_delete_fut = g_name_key.delete_async()
driver_fut = GatewayDriver.unref_async(gateway.driver_hash)
storagetypes.wait_futures([g_name_delete_fut, driver_fut])
Gateway.FlushCache(g_id)
Gateway.FlushCacheDriver(gateway.driver_hash)
g_name_to_id_cache_key = Gateway.Read_ByName_name_cache_key(
g_name_or_id)
storagetypes.memcache.delete(g_name_to_id_cache_key)
return True
def txn():
dk = storagetypes.make_key(
GatewayDriver, GatewayDriver.make_key_name(driver_hash))
d = dk.get()
f = None
if d is None:
d = GatewayDriver(key=dk,
driver_hash=driver_hash,
driver_text=_text,
refcount=1)
d.put()
else:
d.refcount += 1
d.put()
return d
def unref_async( cls, driver_hash ):
"""
Unref a driver, asynchronously
Delete it if its ref count goes non-positive.
Do this in an "outer" transaction (i.e. Gateway.Delete, Gateway.Update)
"""
dk = storagetypes.make_key( GatewayDriver, cls.make_key_name( driver_hash ) )
d = dk.get()
if d is None:
return True
d.ref -= 1
if d.ref <= 0:
d.delete_async()
else:
d.put_async()
return True
def Create(cls, user_cert):
"""
Create a SyndicateUser from a user_cert.
NOTE: the caller will need to have validated the user cert
"""
kwargs = cls.cert_to_dict(user_cert)
email = kwargs['email']
missing = SyndicateUser.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
invalid = SyndicateUser.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# create!
user = SyndicateUser.get_or_insert(user_key_name, **kwargs)
# check for collisions
if user.owner_id != kwargs['owner_id']:
# collision
raise Exception("User '%s' already exists" % email)
return user.key
else:
raise Exception("User '%s' already exists" % email)
else:
raise Exception("User '%s' already exists" % email)
def Create( cls, user_cert ):
"""
Create a SyndicateUser from a user_cert.
NOTE: the caller will need to have validated the user cert
"""
kwargs = cls.cert_to_dict( user_cert )
email = kwargs['email']
missing = SyndicateUser.find_missing_attrs( kwargs )
if len(missing) != 0:
raise Exception( "Missing attributes: %s" % (", ".join( missing )))
invalid = SyndicateUser.validate_fields( kwargs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )))
user_key_name = SyndicateUser.make_key_name( email=email )
user = storagetypes.memcache.get( user_key_name )
if user == None:
user_key = storagetypes.make_key( SyndicateUser, user_key_name )
user = user_key.get()
if user == None:
# create!
user = SyndicateUser.get_or_insert( user_key_name, **kwargs )
# check for collisions
if user.owner_id != kwargs['owner_id']:
# collision
raise Exception("User '%s' already exists" % email)
return user.key
else:
raise Exception("User '%s' already exists" % email)
else:
raise Exception("User '%s' already exists" % email)
def Delete( cls, g_name_or_id ):
"""
Given a gateway ID, delete the corresponding gateway.
That is, set it's "deleted" flag so it no longer gets read.
Unref the driver as well.
"""
gateway = Gateway.Read( g_name_or_id )
if gateway:
g_id = gateway.g_id
else:
raise Exception("No such Gateway '%s'" % g_name_or_id )
key_name = Gateway.make_key_name( g_id=g_id )
def set_deleted():
# atomically set the gateway to deleted
g_key = storagetypes.make_key( cls, key_name )
gw = g_key.get()
if gw is None:
return None
gw.deleted = True
gw.put()
return gw.key
storagetypes.transaction( lambda: set_deleted() )
g_name_key = storagetypes.make_key( GatewayNameHolder, GatewayNameHolder.make_key_name( gateway.name ) )
g_name_delete_fut = g_name_key.delete_async()
driver_fut = GatewayDriver.unref_async( gateway.driver_hash )
storagetypes.wait_futures( [g_name_delete_fut, driver_fut] )
Gateway.FlushCache( g_id )
Gateway.FlushCacheDriver( gateway.driver_hash )
g_name_to_id_cache_key = Gateway.Read_ByName_name_cache_key( g_name_or_id )
storagetypes.memcache.delete( g_name_to_id_cache_key )
return True
def unref_async(cls, driver_hash):
"""
Unref a driver, asynchronously
Delete it if its ref count goes non-positive.
Do this in an "outer" transaction (i.e. Gateway.Delete, Gateway.Update)
"""
dk = storagetypes.make_key(GatewayDriver,
cls.make_key_name(driver_hash))
d = dk.get()
if d is None:
return True
d.refcount -= 1
if d.refcount <= 0:
dk.delete_async()
else:
d.put_async()
return True
def ReadDriver(cls, driver_hash):
"""
Given a driver's hash, return the driver.
"""
driver_hash = driver_hash.lower()
driver_key_name = GatewayDriver.make_key_name(driver_hash)
driver = storagetypes.memcache.get(driver_key_name)
if driver is not None:
return driver
driver_key = storagetypes.make_key(GatewayDriver, driver_key_name)
driver = driver_key.get()
if driver is None:
return None
driver_text = driver.driver_text
if driver is not None:
storagetypes.memcache.set(driver_key_name, driver_text)
return driver_text
def ReadDriver( cls, driver_hash ):
"""
Given a driver's hash, return the driver.
"""
driver_hash = driver_hash.lower()
driver_key_name = GatewayDriver.make_key_name( driver_hash )
driver = storagetypes.memcache.get( driver_key_name )
if driver is not None:
return driver
driver_key = storagetypes.make_key( GatewayDriver, driver_key_name )
driver = driver_key.get()
if driver is None:
return None
driver_text = driver.driver_text
if driver is not None:
storagetypes.memcache.set( driver_key_name, driver_text )
return driver_text
def swap( free_file_id ):
rc, alloced_idx, free_idx_file_id = yield storagetypes.transaction_async( lambda: do_swap( free_file_id ), xg=True )
if rc < 0:
storagetypes.concurrent_return( rc )
old_dir_index = None
if free_file_id is None:
free_file_id = free_idx_file_id
if free_file_id is not None:
# blow away the newly-freed index node
old_entry_idx_key_name = MSEntryEntDirIndex.make_key_name( volume_id, free_file_id )
old_entry_idx_key = storagetypes.make_key( MSEntryEntDirIndex, old_entry_idx_key_name )
yield old_entry_idx_key.delete_async()
storagetypes.memcache.delete( old_entry_idx_key_name )
old_dir_index = alloced_idx.dir_index
storagetypes.concurrent_return( old_dir_index )
try:
g_id = int( g_name_or_id )
except:
gateway_name = g_name_or_id
return cls.Read_ByName( gateway_name, async=async, use_memcache=use_memcache )
key_name = Gateway.make_key_name( g_id=g_id )
g = None
if use_memcache:
g = storagetypes.memcache.get( key_name )
if g == None:
g_key = storagetypes.make_key( cls, Gateway.make_key_name( g_id=g_id ) )
if async:
g_fut = g_key.get_async( use_memcache=False )
return g_fut
else:
g = g_key.get( use_memcache=False )
if not g:
logging.error("Gateway %s not found at all!" % g_id)
elif use_memcache:
storagetypes.memcache.set( key_name, g )
elif async:
def Create( cls, user, volume, **kwargs ):
"""
Create a gateway.
NOTE: careful--caps are required! don't let users call this directly.
"""
# enforce volume ID
kwargs['volume_id'] = volume.volume_id
# enforce ownership--make sure the calling user owns this gateway
kwargs['owner_id'] = user.owner_id
# populate kwargs with default values for missing attrs
cls.fill_defaults( kwargs )
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs( kwargs )
if len(missing) != 0:
raise Exception( "Missing attributes: %s" % (", ".join( missing )))
# sanity check: are our fields valid?
invalid = cls.validate_fields( kwargs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )) )
# what kind of gateway are we?
gateway_type = kwargs['gateway_type']
# set capabilities correctly and safely
kwargs['caps'] = cls.safe_caps( gateway_type, volume.default_gateway_caps )
# ID...
g_id = random.randint( 0, 2**63 - 1 )
kwargs['g_id'] = g_id
g_key_name = Gateway.make_key_name( g_id=g_id )
g_key = storagetypes.make_key( cls, g_key_name )
# create a nameholder and this gateway at once---there's a good chance we'll succeed
gateway_nameholder_fut = GatewayNameHolder.create_async( kwargs['name'], g_id )
gateway_fut = cls.get_or_insert_async( g_key_name, **kwargs )
# wait for operations to complete
storagetypes.wait_futures( [gateway_nameholder_fut, gateway_fut] )
# check for collision...
gateway_nameholder = gateway_nameholder_fut.get_result()
gateway = gateway_fut.get_result()
if gateway_nameholder.g_id != g_id:
# name collision...
storagetypes.deferred.defer( Gateway.delete_all, [g_key] )
raise Exception( "Gateway '%s' already exists!" % kwargs['name'] )
if gateway.g_id != g_id:
# ID collision...
storagetypes.deferred.defer( Gateway.delete_all, [gateway_nameholder.key, g_key] )
raise Exception( "Gateway ID collision. Please try again." )
# we're good!
return g_key
g_id = int(g_name_or_id)
except:
gateway_name = g_name_or_id
return cls.Read_ByName(gateway_name,
async=async,
use_memcache=use_memcache)
key_name = Gateway.make_key_name(g_id=g_id)
g = None
if use_memcache:
g = storagetypes.memcache.get(key_name)
if g == None:
g_key = storagetypes.make_key(cls,
Gateway.make_key_name(g_id=g_id))
if async:
g_fut = g_key.get_async(use_memcache=False)
return g_fut
else:
g = g_key.get(use_memcache=False)
if not g:
logging.error("Gateway %s not found at all!" % g_id)
elif use_memcache:
storagetypes.memcache.set(key_name, g)
elif async:
def Create(cls, user, volume, gateway_cert, driver_text):
"""
Create a gateway, using its user-signed gateway certificate.
NOTE: the caller must verify the authenticity of the certificate.
"""
kwargs = cls.cert_to_dict(gateway_cert)
# sanity check
if kwargs['volume_id'] != volume.volume_id:
raise Exception("Volume ID mismatch: cert has %s; expected %s" %
(kwargs['volume_id'], volume.volume_id))
if kwargs['owner_id'] != user.owner_id:
# this is only okay if the user is the volume owner, and the gateway ID is the anonymous gateway
if not (kwargs['owner_id'] == USER_ID_ANON
and volume.owner_id == user.owner_id):
raise Exception("User ID mismatch: cert has %s; expected %s" %
(kwargs['owner_id'], user.owner_id))
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
# sanity check: are our fields valid?
invalid = cls.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
# sanity check: does the driver match the driver's hash in the cert?
if driver_text is not None:
driver_hash = GatewayDriver.hash_driver(driver_text)
if driver_hash != binascii.hexlify(gateway_cert.driver_hash):
raise Exception(
"Driver hash mismatch: len = %s, expected = %s, got = %s" %
(len(driver_text), driver_hash,
binascii.hexlify(cert.driver_hash)))
gateway_type = kwargs['gateway_type']
# enforce cert distribution
kwargs['need_cert'] = Gateway.needs_cert(gateway_type, kwargs['caps'])
g_id = kwargs['g_id']
g_key_name = Gateway.make_key_name(g_id=g_id)
g_key = storagetypes.make_key(cls, g_key_name)
# create a nameholder and this gateway at once---there's a good chance we'll succeed
futs = []
gateway_nameholder_fut = GatewayNameHolder.create_async(
kwargs['name'], g_id)
gateway_fut = cls.get_or_insert_async(g_key_name, **kwargs)
futs = [gateway_nameholder_fut, gateway_fut]
gateway_driver = None
if driver_text is not None:
gateway_driver = GatewayDriver.create_or_ref(driver_text)
# wait for operations to complete
storagetypes.wait_futures(futs)
# check for collision...
gateway_nameholder = gateway_nameholder_fut.get_result()
gateway = gateway_fut.get_result()
to_rollback = []
if gateway_driver is not None:
to_rollback.append(gateway_driver.key)
if gateway_nameholder.g_id != g_id:
# name collision...
to_rollback.append(g_key)
storagetypes.deferred.defer(Gateway.delete_all, to_rollback)
raise Exception("Gateway '%s' already exists!" % kwargs['name'])
if gateway.g_id != g_id:
# ID collision...
to_rollback.append(gateway_nameholder.key)
to_rollback.append(g_key)
storagetypes.deferred.defer(Gateway.delete_all, to_rollback)
raise Exception("Gateway ID collision. Please try again.")
# we're good!
return g_key
class SyndicateUser(storagetypes.Object):
email = storagetypes.String() # used as the username to Syndicate
owner_id = storagetypes.Integer() # numeric ID for gateways
admin_id = storagetypes.Integer() # which admin made this user?
max_volumes = storagetypes.Integer(
default=10
) # how many Volumes can this user create? (-1 means unlimited)
max_gateways = storagetypes.Integer(
default=10
) # how many gateways can this user create? (-1 means unlimited)
is_admin = storagetypes.Boolean(
default=False, indexed=False) # is this user an administrator?
public_key = storagetypes.Text(
) # PEM-encoded public key for authenticating this user, or USER_KEY_UNSET if it is not set, or USER_KEY_UNUSED if it will not be used
user_cert_protobuf = storagetypes.Blob(
) # protobuf'ed certificate for this user
signature = storagetypes.Blob(
) # signature over the data used to generate this record
# for RPC
key_type = "user"
required_attrs = ["email", "public_key"]
key_attrs = ["email"]
default_values = {
"max_volumes": (lambda cls, attrs: 10),
"max_gateways": (lambda cls, attrs: 10),
"is_admin": (lambda cls, attrs: False),
}
validators = {
"email": (lambda cls, value: valid_email(value)),
"public_key": (lambda cls, value: cls.is_valid_key(
value, USER_RSA_KEYSIZE) and cls.is_public_key(value))
}
read_attrs_api_required = [
"email",
"owner_id",
"max_volumes",
"max_gateways",
"public_key",
]
read_attrs = read_attrs_api_required
write_attrs_api_required = [
"public_key",
]
write_attrs_admin_required = ["max_volumes", "max_gateways", "is_admin"]
write_attrs = write_attrs_api_required + write_attrs_admin_required
# what fields in the cert can change?
modifiable_cert_fields = ["public_key", "max_volumes", "max_gateways"]
def owned_by(self, user):
return user.owner_id == self.owner_id
@classmethod
def Authenticate(cls, email, data, data_signature):
"""
Authenticate a user via public-key cryptography.
Verify that data was signed by the user's private key, given the signature and data.
(use RSA PSS for security).
Return the user on success; False on authentication error; None if the user doesn't exist
"""
user = SyndicateUser.Read(email)
if user == None:
return None
ret = cls.auth_verify(user.public_key, data, data_signature)
if not ret:
logging.error("Verification failed for %s" % email)
return False
else:
return user
@classmethod
def cert_to_dict(cls, user_cert):
attrs = {
'email': str(user_cert.email),
'owner_id': user_cert.user_id,
'public_key': str(user_cert.public_key),
'admin_id': user_cert.admin_id,
'max_volumes': user_cert.max_volumes,
'max_gateways': user_cert.max_gateways,
'is_admin': user_cert.is_admin,
'signature': str(user_cert.signature),
'user_cert_protobuf': user_cert.SerializeToString()
}
return attrs
@classmethod
def Create(cls, user_cert):
"""
Create a SyndicateUser from a user_cert.
NOTE: the caller will need to have validated the user cert
"""
kwargs = cls.cert_to_dict(user_cert)
email = kwargs['email']
missing = SyndicateUser.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
invalid = SyndicateUser.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# create!
user = SyndicateUser.get_or_insert(user_key_name, **kwargs)
# check for collisions
if user.owner_id != kwargs['owner_id']:
# collision
raise Exception("User '%s' already exists" % email)
return user.key
else:
raise Exception("User '%s' already exists" % email)
else:
raise Exception("User '%s' already exists" % email)
@classmethod
def CreateAdmin(cls, email, owner_id, public_key, syndicate_private_key):
"""
Create the admin user.
Called when the MS initializes itself for the first time
"""
import common.api as api
admin_cert = ms_pb2.ms_user_cert()
admin_cert.user_id = owner_id
admin_cert.email = email
admin_cert.public_key = public_key
admin_cert.admin_id = owner_id
admin_cert.max_volumes = -1
admin_cert.max_gateways = -1
admin_cert.is_admin = True
admin_cert.signature = ""
admin_cert_str = admin_cert.SerializeToString()
sig = api.sign_data(syndicate_private_key, admin_cert_str)
admin_cert.signature = base64.b64encode(sig)
return SyndicateUser.Create(admin_cert)
@classmethod
def Read(cls, email_or_owner_id, async=False):
"""
Read a SyndicateUser
Arguments:
email_or_owner_id -- Email address of the user to read, or the owner ID (str or int)
"""
owner_id = None
email = None
try:
owner_id = int(email_or_owner_id)
except:
email = email_or_owner_id
if owner_id is not None:
return cls.Read_ByOwnerID(owner_id, async=async)
user_key_name = SyndicateUser.make_key_name(email=email)
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = storagetypes.memcache.get(user_key_name)
if user == None:
if async:
return user_key.get_async(use_memcache=False)
else:
user = user_key.get(use_memcache=False)
if not user:
return None
else:
storagetypes.memcache.set(user_key_name, user)
elif async:
user = storagetypes.FutureWrapper(user)
return user
def Create( cls, user, volume, gateway_cert, driver_text ):
"""
Create a gateway, using its user-signed gateway certificate.
NOTE: the caller must verify the authenticity of the certificate.
"""
kwargs = cls.cert_to_dict( gateway_cert )
# sanity check
if kwargs['volume_id'] != volume.volume_id:
raise Exception("Volume ID mismatch: cert has %s; expected %s" % (kwargs['volume_id'], volume.volume_id))
if kwargs['owner_id'] != user.owner_id:
raise Exception("User ID mismatch: cert has %s; expected %s" % (kwargs['owner_id'], user.owner_id) )
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs( kwargs )
if len(missing) != 0:
raise Exception( "Missing attributes: %s" % (", ".join( missing )))
# sanity check: are our fields valid?
invalid = cls.validate_fields( kwargs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )) )
# sanity check: does the driver match the driver's hash in the cert?
if driver_text is not None:
driver_hash = GatewayDriver.hash_driver( driver_text )
if driver_hash != binascii.hexlify( gateway_cert.driver_hash ):
raise Exception("Driver hash mismatch: len = %s, expected = %s, got = %s" % (len(driver_text), driver_hash, binascii.hexlify( cert.driver_hash )))
gateway_type = kwargs['gateway_type']
# enforce cert distribution
kwargs['need_cert'] = Gateway.needs_cert( gateway_type, kwargs['caps'] )
g_id = kwargs['g_id']
g_key_name = Gateway.make_key_name( g_id=g_id )
g_key = storagetypes.make_key( cls, g_key_name )
# create a nameholder and this gateway at once---there's a good chance we'll succeed
futs = []
gateway_nameholder_fut = GatewayNameHolder.create_async( kwargs['name'], g_id )
gateway_fut = cls.get_or_insert_async( g_key_name, **kwargs )
futs = [gateway_nameholder_fut, gateway_fut]
gateway_driver = None
if driver_text is not None:
gateway_driver = GatewayDriver.create_or_ref( driver_text )
# wait for operations to complete
storagetypes.wait_futures( futs )
# check for collision...
gateway_nameholder = gateway_nameholder_fut.get_result()
gateway = gateway_fut.get_result()
to_rollback = []
if gateway_driver is not None:
to_rollback.append( gateway_driver.key )
if gateway_nameholder.g_id != g_id:
# name collision...
to_rollback.append( g_key )
storagetypes.deferred.defer( Gateway.delete_all, to_rollback )
raise Exception( "Gateway '%s' already exists!" % kwargs['name'] )
if gateway.g_id != g_id:
# ID collision...
to_rollback.append( gateway_nameholder.key )
to_rollback.append( g_key )
storagetypes.deferred.defer( Gateway.delete_all, to_rollback )
raise Exception( "Gateway ID collision. Please try again." )
# we're good!
return g_key
gateway_name = g_name_or_id
return cls.Read_ByName(gateway_name,
async=async,
use_memcache=use_memcache)
key_name = Gateway.make_key_name(g_id=g_id)
g = None
if use_memcache:
g = storagetypes.memcache.get(key_name)
if g is not None and not deleted and g.deleted:
storagetypes.memcache.delete(key_name)
g = None
if g is None:
g_key = storagetypes.make_key(cls,
Gateway.make_key_name(g_id=g_id))
if async:
g_fut = cls.Read_Async(g_key, deleted=deleted)
return g_fut
else:
g = g_key.get(use_memcache=False)
if g is None:
logging.error("Gateway %s not found at all!" % g_id)
if g.deleted:
g = None
elif use_memcache and g is not None:
def Create(cls, email, **kwargs):
"""
Create a SyndicateUser.
Required arguments:
email -- Email address of the user. Serves as the username (str)
openid_url -- OpenID identifier for authenticating this user (str)
"""
kwargs['email'] = email
# sanity check
SyndicateUser.fill_defaults(kwargs)
# if we're given a signing public key, then set it.
# otherwise, use the given salted password hash.
skip_verify = []
if kwargs.has_key('activate_password_hash') and kwargs.has_key(
'activate_password_salt'):
# don't check for this
skip_verify = ['signing_public_key']
kwargs['signing_public_key'] = cls.USER_KEY_UNSET
elif kwargs.has_key('signing_public_key'):
# this had better be a valid key
if not SyndicateUser.validators['signing_public_key'](
SyndicateUser, kwargs['signing_public_key']):
raise Exception("Invalid field: %s" % 'signing_public_key')
# don't check for password hash and salt
skip_verify = ['activate_password_hash', 'activate_password_salt']
else:
# need either of these...
raise Exception(
"Need either signing_public_key or (activate_password_hash, activate_password_salt)"
)
missing = SyndicateUser.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
invalid = SyndicateUser.validate_fields(kwargs, skip=skip_verify)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# do not allow admin privileges
kwargs['is_admin'] = False
kwargs['owner_id'] = random.randint(1, 2**63 - 1)
user_key_name = SyndicateUser.make_key_name(email=email)
user = SyndicateUser.get_or_insert(user_key_name, **kwargs)
# check for collisions
if user.owner_id != kwargs['owner_id']:
# collision
raise Exception("User '%s' already exists" % email)
return user.key
else:
raise Exception("User '%s' already exists" % email)
else:
raise Exception("User '%s' already exists" % email)
g_id = int( g_name_or_id )
except:
gateway_name = g_name_or_id
return cls.Read_ByName( gateway_name, async=async, use_memcache=use_memcache )
key_name = Gateway.make_key_name( g_id=g_id )
g = None
if use_memcache:
g = storagetypes.memcache.get( key_name )
if g is not None and not deleted and g.deleted:
storagetypes.memcache.delete( key_name )
g = None
if g is None:
g_key = storagetypes.make_key( cls, Gateway.make_key_name( g_id=g_id ) )
if async:
g_fut = cls.Read_Async( g_key, deleted=deleted )
return g_fut
else:
g = g_key.get( use_memcache=False )
if g is None:
logging.error("Gateway %s not found at all!" % g_id)
if g.deleted:
g = None
elif use_memcache and g is not None:
closure_id = int(closure_name_or_id)
except:
closure_name = closure_name
return cls.Read_ByName(closure_name,
async=async,
use_memcache=use_memcache)
key_name = Closure.make_key_name(closure_id=closure_id)
closure = None
if use_memcache:
closure = storagetypes.memcache.get(key_name)
if closure is None:
c_key = storagetypes.make_key(
cls, Closure.make_key_name(closure_id=closure_id))
if async:
c_fut = c_key.get_async(use_memcache=False)
return c_fut
else:
closure = c_key.get(use_memcache=False)
if closure is None:
logging.error("Closure %s not found at all!" % closure_id)
elif use_memcache:
storagetypes.memcache.set(key_name, closure)
elif async:
use_memcache -- If True, check memcache for the Volume, and if async is false, cache the results.
"""
volume_id = None
volume_name = None
try:
volume_id = int(volume_name_or_id)
except:
volume_name = volume_name_or_id
return cls.Read_ByName(volume_name,
async=async,
use_memcache=use_memcache)
volume_key_name = Volume.make_key_name(volume_id=volume_id)
volume_key = storagetypes.make_key(Volume, volume_key_name)
volume = storagetypes.memcache.get(volume_key_name)
if volume == None:
if async:
return volume_key.get_async(use_memcache=False)
else:
volume = volume_key.get(use_memcache=False)
if not volume:
return None
else:
storagetypes.memcache.set(volume_key_name, volume)
elif async:
volume = storagetypes.FutureWrapper(volume)
class SyndicateUser(storagetypes.Object):
USER_KEY_UNSET = "unset"
USER_KEY_UNUSED = "unused"
email = storagetypes.String() # used as the username
owner_id = storagetypes.Integer() # UID field in Syndicate
openid_url = storagetypes.Text() # OpenID identifying URL
max_volumes = storagetypes.Integer(
default=10
) # how many Volumes can this user create? (-1 means unlimited)
max_UGs = storagetypes.Integer(
default=10) # how many UGs can this user create?
max_RGs = storagetypes.Integer(
default=10) # how many RGs can this user create?
max_AGs = storagetypes.Integer(
default=10) # how many AGs can this user create?
max_requests = storagetypes.Integer(
default=10) # how many pending Volume requests can this user create?
is_admin = storagetypes.Boolean(
default=False, indexed=False) # is this user an administrator?
signing_public_key = storagetypes.Text(
) # PEM-encoded public key for authenticating this user, or USER_KEY_UNSET if it is not set, or USER_KEY_UNUSED if it will not be used
signing_public_key_expiration = storagetypes.Integer(
default=-1) # seconds since the epoch
active = storagetypes.Boolean(default=False) # is this account active?
allow_password_auth = storagetypes.Boolean(
default=True) # allow password-based authentication?
# one-time password for setting the signing public key
activate_password_salt = storagetypes.String(
) # 32 bytes, but encoded as a hex string
activate_password_hash = storagetypes.String() # SHA256
# for RPC
key_type = "user"
required_attrs = ["email", "openid_url", "signing_public_key_expiration"]
key_attrs = ["email"]
default_values = {
"max_volumes": (lambda cls, attrs: 10),
"max_UGs": (lambda cls, attrs: 10),
"max_RGs": (lambda cls, attrs: 10),
"max_AGs": (lambda cls, attrs: 10),
"is_admin": (lambda cls, attrs: False),
"openid_url": (lambda cls, attrs: ""),
"signing_public_key_expiration": (lambda cls, attrs: -1),
"active": (lambda cls, attrs: False),
"allow_password_auth": (lambda cls, attrs: True)
}
validators = {
"email": (lambda cls, value: valid_email(cls, value)),
"signing_public_key":
(lambda cls, value: not cls.is_signing_public_key_set(value) or cls.
is_valid_key(value, USER_RSA_KEYSIZE)),
"openid_url":
(lambda cls, value: len(value) < 4096), # not much of a check here...
"activate_password_salt": (lambda cls, value: len(
str(value).translate(None, "0123456789abcdefABCDEF")) == 0 and len(
str(value)) == 64), # 32-byte salt, encoded as a hex number
"activate_password_hash":
(lambda cls, value:
len(str(value).translate(None, "0123456789abcdefABCDEF")
) == 0 and len(str(value)) == 64
) # SHA256: 32-byte hash, encoded as a hex number
}
read_attrs_api_required = [
"email",
"owner_id",
"openid_url",
"max_volumes",
"max_UGs",
"max_RGs",
"max_AGs",
"signing_public_key",
"signing_public_key_expiration",
]
read_attrs = read_attrs_api_required
write_attrs_api_required = [
"openid_url", "signing_public_key", "allow_password_auth"
]
write_attrs_admin_required = [
"max_volumes", "max_UGs", "max_RGs", "max_AGs", "is_admin"
]
write_attrs = write_attrs_api_required + write_attrs_admin_required
def owned_by(self, user):
return user.owner_id == self.owner_id
@classmethod
def Authenticate(cls, email, data, data_signature):
"""
Authenticate a user via public-key cryptography.
Verify that data was signed by the user's private key, given the signature and data.
(use RSA PSS for security).
Return the user on success; False on authentication error; None if the user doesn't exist
"""
user = SyndicateUser.Read(email)
if user == None:
return None
if not SyndicateUser.is_signing_public_key_set(
user.signing_public_key):
logging.error("Key for %s is not set or unused" % email)
return None
ret = cls.auth_verify(user.signing_public_key, data, data_signature)
if not ret:
logging.error("Verification failed for %s" % email)
return False
else:
return user
def makeCert(self):
ret = {}
ret['expires'] = self.signing_public_key_expiration
ret['pubkey'] = self.signing_public_key
ret['email'] = self.email
ret['openid_url'] = self.openid_url
return ret
@classmethod
def Create(cls, email, **kwargs):
"""
Create a SyndicateUser.
Required arguments:
email -- Email address of the user. Serves as the username (str)
openid_url -- OpenID identifier for authenticating this user (str)
"""
kwargs['email'] = email
# sanity check
SyndicateUser.fill_defaults(kwargs)
# if we're given a signing public key, then set it.
# otherwise, use the given salted password hash.
skip_verify = []
if kwargs.has_key('activate_password_hash') and kwargs.has_key(
'activate_password_salt'):
# don't check for this
skip_verify = ['signing_public_key']
kwargs['signing_public_key'] = cls.USER_KEY_UNSET
elif kwargs.has_key('signing_public_key'):
# this had better be a valid key
if not SyndicateUser.validators['signing_public_key'](
SyndicateUser, kwargs['signing_public_key']):
raise Exception("Invalid field: %s" % 'signing_public_key')
# don't check for password hash and salt
skip_verify = ['activate_password_hash', 'activate_password_salt']
else:
# need either of these...
raise Exception(
"Need either signing_public_key or (activate_password_hash, activate_password_salt)"
)
missing = SyndicateUser.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
invalid = SyndicateUser.validate_fields(kwargs, skip=skip_verify)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# do not allow admin privileges
kwargs['is_admin'] = False
kwargs['owner_id'] = random.randint(1, 2**63 - 1)
user_key_name = SyndicateUser.make_key_name(email=email)
user = SyndicateUser.get_or_insert(user_key_name, **kwargs)
# check for collisions
if user.owner_id != kwargs['owner_id']:
# collision
raise Exception("User '%s' already exists" % email)
return user.key
else:
raise Exception("User '%s' already exists" % email)
else:
raise Exception("User '%s' already exists" % email)
@classmethod
def CreateAdmin(cls, email, openid_url, signing_public_key,
activate_password):
"""
Create the Admin user. NOTE: this will be called repeatedly, so use memcache
"""
user_key_name = SyndicateUser.make_key_name(email=email)
user = storagetypes.memcache.get(user_key_name)
if user == None:
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = user_key.get()
if user == None:
# admin does not exist
attrs = {}
logging.info("Generating admin '%s'" % email)
# fill defaults
SyndicateUser.fill_defaults(attrs)
attrs['email'] = email
attrs['openid_url'] = openid_url
attrs['owner_id'] = random.randint(1, 2**63 - 1)
attrs['is_admin'] = True
# generate password hash and salt
import common.api as api
pw_salt = api.password_salt()
pw_hash = api.hash_password(activate_password, pw_salt)
attrs['activate_password_hash'] = pw_hash
attrs['activate_password_salt'] = pw_salt
# possible that we haven't set the public key yet
if not signing_public_key or len(signing_public_key) == 0:
signing_public_key = cls.USER_KEY_UNSET
attrs['signing_public_key'] = signing_public_key
invalid = SyndicateUser.validate_fields(attrs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
user = SyndicateUser.get_or_insert(user_key_name, **attrs)
# check for collisions
if user.owner_id != attrs['owner_id']:
# collision
logging.warning("Admin '%s' already exists" % email)
storagetypes.memcache.set(user_key_name, user)
return user.key
@classmethod
def Read(cls, email_or_owner_id, async=False):
"""
Read a SyndicateUser
Arguments:
email_or_owner_id -- Email address of the user to read, or the owner ID (str or int)
"""
owner_id = None
email = None
try:
owner_id = int(email_or_owner_id)
except:
email = email_or_owner_id
if owner_id != None:
return cls.Read_ByOwnerID(owner_id, async=async)
user_key_name = SyndicateUser.make_key_name(email=email)
user_key = storagetypes.make_key(SyndicateUser, user_key_name)
user = storagetypes.memcache.get(user_key_name)
if user == None:
if async:
return user_key.get_async(use_memcache=False)
else:
user = user_key.get(use_memcache=False)
if not user:
return None
else:
storagetypes.memcache.set(user_key_name, user)
elif async:
user = storagetypes.FutureWrapper(user)
return user
if idx.dir_index != index:
storagetypes.concurrent_return( (-errno.EPERM, None) )
storagetypes.concurrent_return( (0, idx) )
@classmethod
def __read_dirent_node( cls, volume_id, parent_id, file_id, index, async=False, check_file_id=True ):
"""
Read a node key, and verify that it is consistent.
Return (rc, idx)
"""
idx_key_name = MSEntryDirEntIndex.make_key_name( volume_id, parent_id, index )
idx_key = storagetypes.make_key( MSEntryDirEntIndex, idx_key_name )
ret_fut = cls.__read_node( file_id, index, idx_key, check_file_id=check_file_id )
if async:
return ret_fut
else:
storagetypes.wait_futures( [ret_fut] )
return ret_fut.get_result()
@classmethod
def __compactify_get_candidates_delete( cls, volume_id, parent_id, dir_index_cutoff, async=False ):
"""
Find the set of allocated index nodes beyond a given offset, suitable for swapping into a newly-freed slot.
"""
def Create(cls, user, volume, **kwargs):
"""
Create a gateway.
NOTE: careful--caps are required! don't let users call this directly.
"""
# enforce volume ID
kwargs['volume_id'] = volume.volume_id
# enforce ownership--make sure the calling user owns this gateway
kwargs['owner_id'] = user.owner_id
# populate kwargs with default values for missing attrs
cls.fill_defaults(kwargs)
# sanity check: do we have everything we need?
missing = cls.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
# sanity check: are our fields valid?
invalid = cls.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
# what kind of gateway are we?
gateway_type = kwargs['gateway_type']
# set capabilities correctly and safely
kwargs['caps'] = cls.safe_caps(gateway_type,
volume.default_gateway_caps)
# enforce cert generation
kwargs['need_cert'] = Gateway.needs_cert(gateway_type, kwargs['caps'])
# ID...
g_id = random.randint(0, 2**63 - 1)
kwargs['g_id'] = g_id
g_key_name = Gateway.make_key_name(g_id=g_id)
g_key = storagetypes.make_key(cls, g_key_name)
# create a nameholder and this gateway at once---there's a good chance we'll succeed
gateway_nameholder_fut = GatewayNameHolder.create_async(
kwargs['name'], g_id)
gateway_fut = cls.get_or_insert_async(g_key_name, **kwargs)
# wait for operations to complete
storagetypes.wait_futures([gateway_nameholder_fut, gateway_fut])
# check for collision...
gateway_nameholder = gateway_nameholder_fut.get_result()
gateway = gateway_fut.get_result()
if gateway_nameholder.g_id != g_id:
# name collision...
storagetypes.deferred.defer(Gateway.delete_all, [g_key])
raise Exception("Gateway '%s' already exists!" % kwargs['name'])
if gateway.g_id != g_id:
# ID collision...
storagetypes.deferred.defer(Gateway.delete_all,
[gateway_nameholder.key, g_key])
raise Exception("Gateway ID collision. Please try again.")
# we're good!
return g_key
def Create( cls, user, **kwargs ):
"""
Given volume data, store it.
Update the corresponding SyndicateUser atomically along with creating the Volume
so that the SyndicateUser owns the Volume.
Arguments:
user -- SyndicateUser instance that will own this Volume
Required keyword arguments:
name -- name of the Volume (str)
blocksize -- size of the Volume's blocks in bytes (int)
description -- description of the Volume (str)
private -- whether or not this Volume is visible to other users (bool)
Optional keyword arguments:
metadata_private_key -- PEM-encoded RSA private key, 4096 bits (str)
archive -- whether or not this Volume is populated only by Acquisition Gateways (bool)
default_gateway_caps -- bitfield of capabilities Gateways created within this Volume should receive
"""
# sanity check
if not user:
raise Exception( "No user given" )
kwargs['owner_id'] = 0 # will look up user and fill with owner ID once we validate input.
Volume.fill_defaults( kwargs )
# extract public key from private key if needed
Volume.extract_keys( 'metadata_public_key', 'metadata_private_key', kwargs, VOLUME_RSA_KEYSIZE )
# Validate
missing = Volume.find_missing_attrs( kwargs )
if len(missing) != 0:
raise Exception( "Missing attributes: %s" % (", ".join( missing )))
invalid = Volume.validate_fields( kwargs )
if len(invalid) != 0:
raise Exception( "Invalid values for fields: %s" % (", ".join( invalid )) )
# vet the keys
for key_field in ['metadata_public_key', 'metadata_private_key']:
key_str = kwargs[key_field]
valid = cls.is_valid_key( key_str, VOLUME_RSA_KEYSIZE )
if not valid:
raise Exception("Key must be a %s-bit RSA key" % (VOLUME_RSA_KEYSIZE) )
# attempt to create the Volume
volume_id = random.randint( 1, 2**63 - 1 )
volume_key_name = Volume.make_key_name( volume_id=volume_id )
volume_key = storagetypes.make_key( Volume, volume_key_name )
# put the Volume and nameholder at the same time---there's a good chance we'll succeed
volume_nameholder_fut = VolumeNameHolder.create_async( kwargs['name'], volume_id )
volume_fut = Volume.get_or_insert_async( volume_key_name,
name=kwargs['name'],
blocksize=kwargs['blocksize'],
description=kwargs['description'],
owner_id=user.owner_id,
volume_id=volume_id,
active=kwargs.get('active',False),
version=1,
cert_version=1,
private=kwargs['private'],
archive=kwargs['archive'],
allow_anon = kwargs['allow_anon'],
metadata_public_key = kwargs['metadata_public_key'],
metadata_private_key = kwargs['metadata_private_key'],
default_gateway_caps = kwargs['default_gateway_caps']
)
storagetypes.wait_futures( [volume_nameholder_fut, volume_fut] )
# verify that there was no collision
volume = volume_fut.get_result()
volume_nameholder = volume_nameholder_fut.get_result()
if volume_nameholder.volume_id != volume_id:
# name collision
storagetypes.deferred.defer( Volume.delete_all, [volume_key] )
raise Exception( "Volume '%s' already exists!" % kwargs['name'])
if volume.volume_id != volume_id:
# ID collision
storagetypes.deferred.defer( Volume.delete_all, [volume_key, volume_nameholder.key] )
raise Exception( "Volume ID collision. Please try again" )
# set permissions
req = VolumeAccessRequest.create_async( user.owner_id, volume_id, kwargs['name'], random.randint(-2**63, 2**63 - 1), VolumeAccessRequest.STATUS_GRANTED,
gateway_caps=kwargs['default_gateway_caps'], allowed_gateways=(1 << GATEWAY_TYPE_AG)|(1 << GATEWAY_TYPE_UG)|(1 << GATEWAY_TYPE_RG), request_message="Created").get_result()
return volume_key
def Create(cls, user, **kwargs):
"""
Given volume data, store it.
Update the corresponding SyndicateUser atomically along with creating the Volume
so that the SyndicateUser owns the Volume.
Arguments:
user -- SyndicateUser instance that will own this Volume
Required keyword arguments:
name -- name of the Volume (str)
blocksize -- size of the Volume's blocks in bytes (int)
description -- description of the Volume (str)
private -- whether or not this Volume is visible to other users (bool)
Optional keyword arguments:
metadata_private_key -- PEM-encoded RSA private key, 4096 bits (str)
archive -- whether or not this Volume is populated only by Acquisition Gateways (bool)
default_gateway_caps -- bitfield of capabilities Gateways created within this Volume should receive
"""
# sanity check
if not user:
raise Exception("No user given")
kwargs[
'owner_id'] = 0 # will look up user and fill with owner ID once we validate input.
Volume.fill_defaults(kwargs)
# extract public key from private key if needed
Volume.extract_keys('metadata_public_key', 'metadata_private_key',
kwargs, VOLUME_RSA_KEYSIZE)
# Validate
missing = Volume.find_missing_attrs(kwargs)
if len(missing) != 0:
raise Exception("Missing attributes: %s" % (", ".join(missing)))
invalid = Volume.validate_fields(kwargs)
if len(invalid) != 0:
raise Exception("Invalid values for fields: %s" %
(", ".join(invalid)))
# vet the keys
for key_field in ['metadata_public_key', 'metadata_private_key']:
key_str = kwargs[key_field]
valid = cls.is_valid_key(key_str, VOLUME_RSA_KEYSIZE)
if not valid:
raise Exception("Key must be a %s-bit RSA key" %
(VOLUME_RSA_KEYSIZE))
# attempt to create the Volume
volume_id = random.randint(1, 2**63 - 1)
volume_key_name = Volume.make_key_name(volume_id=volume_id)
volume_key = storagetypes.make_key(Volume, volume_key_name)
# put the Volume and nameholder at the same time---there's a good chance we'll succeed
volume_nameholder_fut = VolumeNameHolder.create_async(
kwargs['name'], volume_id)
volume_fut = Volume.get_or_insert_async(
volume_key_name,
name=kwargs['name'],
blocksize=kwargs['blocksize'],
description=kwargs['description'],
owner_id=user.owner_id,
volume_id=volume_id,
active=kwargs.get('active', False),
version=1,
cert_version=1,
private=kwargs['private'],
archive=kwargs['archive'],
allow_anon=kwargs['allow_anon'],
metadata_public_key=kwargs['metadata_public_key'],
metadata_private_key=kwargs['metadata_private_key'],
default_gateway_caps=kwargs['default_gateway_caps'])
storagetypes.wait_futures([volume_nameholder_fut, volume_fut])
# verify that there was no collision
volume = volume_fut.get_result()
volume_nameholder = volume_nameholder_fut.get_result()
if volume_nameholder.volume_id != volume_id:
# name collision
storagetypes.deferred.defer(Volume.delete_all, [volume_key])
raise Exception("Volume '%s' already exists!" % kwargs['name'])
if volume.volume_id != volume_id:
# ID collision
storagetypes.deferred.defer(Volume.delete_all,
[volume_key, volume_nameholder.key])
raise Exception("Volume ID collision. Please try again")
# set permissions
req = VolumeAccessRequest.create_async(
user.owner_id,
volume_id,
kwargs['name'],
random.randint(-2**63, 2**63 - 1),
VolumeAccessRequest.STATUS_GRANTED,
gateway_caps=kwargs['default_gateway_caps'],
allowed_gateways=(1 << GATEWAY_TYPE_AG) | (1 << GATEWAY_TYPE_UG) |
(1 << GATEWAY_TYPE_RG),
request_message="Created").get_result()
return volume_key
