def __init__(self, store, auth_user):
'''param store: an instance of GenericServerStore, could be in memory for testing or
MongoStore.
param auth_user: the user invoking this service, must be prior authenticated by app'''
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
def test_publish_dev_with_tag(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False))
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (['mock_id'], [], [], [])
block.deltas = []
ensure = Security('authUser', store)
ensure.check_read_block = Mock(return_value=True)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_publish_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.deptable = BlockVersionTable()
p.publish(pack)
def test_check_make_private_block(self):
# 0. Goku is paying
self._subscribe("goku", "enterprise_275_50_x")
# 1. Already private.
brl_master = BRLBlock("goku/goku/block/master")
bper_master = ElementPermissions(brl_master, private=True)
self.store.upsert_block_permissions(bper_master)
# 2. Bulma cant make it private even its already private because is not an admin
ensure = Security("bulma", self.store)
self._subscribe("bulma", "personal_7_1_x")
self.assertRaises(ForbiddenException, ensure.check_make_private_a_block, "goku")
# 3. Now block is public, so bulma can't make it private
bper_master = ElementPermissions(brl_master, private=False)
self.store.upsert_block_permissions(bper_master)
self.assertRaises(ForbiddenException, ensure.check_make_private_a_block, "goku")
# 4. Grant read and write, and bulma still can't make it private
bper_master.read.grant("bulma")
bper_master.write.grant("bulma")
self.store.upsert_block_permissions(bper_master)
self.assertRaises(ForbiddenException, ensure.check_make_private_a_block, "goku")
# 5. Make bulma an admin, now bulma can make it private
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_make_private_a_block("goku")
# 6. Bulma cant make private the block when suscription is not active
self._subscribe("goku", "free")
self.assertRaises(ForbiddenException, ensure.check_make_private_a_block, "goku")
def test_check_make_public_block(self):
# 1. Already public.
brl_master = BRLBlock("goku/goku/block/master")
bper_master = ElementPermissions(brl_master, private=False)
self.store.upsert_block_permissions(bper_master)
# 2. Bulma cant make it public even its already public
ensure = Security("bulma", self.store)
self._subscribe("bulma", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_make_private_a_block, "goku")
# 3. Now block is public, so bulma cant make it public
bper_master = ElementPermissions(brl_master, private=True)
self.store.upsert_block_permissions(bper_master)
self.assertRaises(ForbiddenException, ensure.check_make_public_a_block, "goku")
# 4. Grant read and write, and bulma still cant make it public
bper_master.read.grant("bulma")
bper_master.write.grant("bulma")
self.store.upsert_block_permissions(bper_master)
self.assertRaises(ForbiddenException, ensure.check_make_public_a_block, "goku")
# 5. Make bulma an admin, now bulma can make it public
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_make_public_a_block("goku")
# 5. Bulma can make public the block even when suscription is not active
self._subscribe("bulma", "free")
goku = self.store.read_user("goku")
self.store.update_user(goku)
ensure.check_make_public_a_block("goku")
def test_publish(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
# moduleID=BlockID(UserID(123),456)
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False))
user = User("owner")
user.numeric_id = 1
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.last_version.return_value = Mock(BlockVersion)
block.add_publication.return_value = (Mock(list), Mock(list), Mock(list), Mock(list))
block.deltas = []
ensure = Security('authUser', store)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_read_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r2.h'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r3.h'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
p.publish(pack)
block.add_publication.assert_called_once_with(pack, p.auth_user)
store.update_block.assert_called_once_with(block)
self.assertEqual(1, store.create_published_cells.call_count)
self.assertEqual(1, store.create_published_contents.call_count)
# Check sizes
self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3
# Publish again, see the size incremented
pack._bytes = None # Lazy computed
p.publish(pack)
self.assertEquals(user.blocks_bytes, 24) # 24 bytes: "hola" * 3 * 2 publications
# Max size exceeded for user
user.max_workspace_size = 25
self.assertRaises(ForbiddenException, p.publish, pack)
# Try to publish only 1 byte
pack._bytes = None # Lazy computed
pack.cells = []
pack.contents = {}
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('h'))
p.publish(pack)
def test_check_read_user_subscription(self):
# Can do it himself (always, event not paying) and not their administrators
ensure = Security("goku", self.store)
self._subscribe("goku", "enterprise_275_50_x")
ensure.check_read_user_subscription("goku")
self._subscribe("goku", "free")
ensure.check_read_user_subscription("goku")
# Other admin-granted user cant do it
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure = Security("bulma", self.store)
ensure.check_read_user_subscription("goku")
def test_update_user(self):
# Can do it himself and their administrators
ensure = Security("goku", self.store)
self._subscribe("goku", "enterprise_275_50_x")
ensure.check_update_user("goku")
self._subscribe("goku", "free")
ensure.check_update_user("goku")
# Other admin-granted user can't do it
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure = Security("bulma", self.store)
ensure.check_update_user("goku")
def test_publish_dev_with_tag(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(
return_value=ElementPermissions(brl, private=False))
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (['mock_id'], [], [], [])
block.deltas = []
ensure = Security('authUser', store)
ensure.check_read_block = Mock(return_value=True)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_publish_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.deptable = BlockVersionTable()
p.publish(pack)
class ReferenceTranslatorService(object):
def __init__(self, store, auth_user):
self._store = store
self.security = Security(auth_user, self._store)
def get_published_resources(self, references):
result = ReferencedResources()
for block_version, cell_names in references.iteritems():
try:
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
cell_ids = block.cells.get_ids(cell_names, block_version.time)
content_ids = block.contents.get_ids(cell_names, block_version.time)
cells = self._store.read_published_cells(cell_ids.values())
contents = self._store.read_published_contents(content_ids.values())
for name, rID in cell_ids.iteritems():
if name in content_ids:
cid = content_ids[name]
cid = contents[cid]
else:
cid = None # Virtual resource
result[block_version][name] = Resource(cells[rID], cid)
except (ForbiddenException, NotInStoreException):
pass
return result
def get_published_min_refs(self, references):
"""returns the minimum information required to perform a compatibility check for those
references. This method is currently used just by CompatibilityClosureBuilder
param references: {block_version: set(cell_names)}
return: {block_version: {cell_name: (cell_id, content_id), root_id, [deps blockcellnames]}}
"""
result = defaultdict(dict)
for block_version, cell_names in references.iteritems():
try:
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
cell_ids = block.cells.get_ids(cell_names, block_version.time)
content_ids = block.contents.get_ids(cell_names, block_version.time)
cells = self._store.read_min_cells(cell_ids.values())
# This cells are {cellID: (rootID, dep_block_names)}
for cell_name, cell_id in cell_ids.iteritems():
content_id = content_ids.get(cell_name) # None if Virtual resource
root_id, deps = cells.get(cell_id, (None, None))
if root_id is not None:
result[block_version][cell_name] = ((cell_id, content_id), root_id, deps)
except (ForbiddenException, NotInStoreException):
pass
return result
def get_dep_table(self, block_version):
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
table = block.dep_tables.find(block_version.time)
return table
class PublishService(object):
''' Service for publish blocks in server.'''
def __init__(self, store, auth_user):
self._store = store
self.auth_user = auth_user
self.security = Security(self.auth_user, self._store)
def publish(self, publish_request):
'''Performs a publication
TIP: If we add publish_request to transaction_definition we can easily have asynchronous
publications
private: Only for first publication
'''
from biicode.server.background.enqueuer import register_publish
if publish_request.tag == DEV:
if not publish_request:
raise BiiRequestErrorException('Up to date, nothing to publish')
if publish_request.versiontag is not None:
raise PublishException('A DEV version cannot have tag %s' % publish_request.tag)
assert publish_request.deptable is not None
# by default it is public
# TODO: BLock creation is not handled in the transaction
target_version = publish_request.parent
user = self._store.read_user(target_version.block.owner)
# Look if user has the block already created, because the block
# can exist with -1 version if it has been created in web
if target_version.block not in user.blocks.keys():
try:
if target_version != publish_request.parent: # Branching
user = self.create_block(target_version.block,
publish_request.parent, private=False)
else:
user = self.create_block(target_version.block, private=False)
except DuplicateBlockException:
pass # Its ok, already created
target_block = target_version.block
self._store.requestBlockTransaction(target_block)
try:
# If we can't read the block, we can't know about his existence
self.security.check_read_block(target_block)
self.security.check_publish_block(target_block, publish_request)
# biiresponse.debug('Read block "%s"' % brl_block)
block = self._store.read_block(target_block)
(cells, contents,
old_cells_ids, old_content_ids) = self._in_memory_block_update(block, publish_request)
except ForbiddenException:
self._store.finishBlockTransaction(target_block)
raise
except PublishException as e:
self._store.finishBlockTransaction(target_block)
raise ServerInternalErrorException(e.message)
except Exception as excp:
logger.error("Exception in publish service!!: %s " % str(excp))
tb = traceback.format_exc()
logger.error(tb)
self._store.finishBlockTransaction(target_block)
raise ServerInternalErrorException()
self._store.beginBlockTransaction(target_block, cells, contents)
try:
self._write_resources_to_db(cells, contents, old_cells_ids, old_content_ids)
self._store.update_block(block)
self._store.commitBlockTransaction(target_block)
register_publish(self.auth_user, block.last_version())
self._store.finishBlockTransaction(target_block)
# Need to read user again, otherwise will raise MongoNotCurrentObjectException
# because of double update of same memory object
user = self._store.read_user(target_version.block.owner)
user.add_block_size_bytes(target_version.block, publish_request.bytes)
# Save user (with block bytes updated)
self._store.update_user(user)
return block.last_version()
except Exception as excp:
tb = traceback.format_exc()
logger.debug(tb)
self._rollback_transaction(excp, target_block)
raise ServerInternalErrorException('Publish transaction failed. Please, retry')
def create_block(self, brl, private=False):
'''Creates a block in server due the brl and description'''
self.security.check_create_block(brl.owner, private)
user = self._store.read_user(brl.owner)
try:
block_id = user.add_block(brl) # should fail if existing
except DuplicateBlockException:
logger.debug('Block %s already existing, not creating it' % brl)
raise
block = Block(block_id, brl)
try: # FIXME: better upsert?
self._store.create_block(block, private) # should fail if existing
except AlreadyInStoreException:
pass
self._store.update_user(user) # raise exception if not current
return user
def _rollback_transaction(self, excp, brl_block):
'''rollback transaction for publish'''
logger.warning(str(excp) + '\nRolling back publish transaction')
self._store.rollBackBlockTransaction(brl_block)
self._store.finishBlockTransaction(brl_block)
def _write_resources_to_db(self, cells, contents, old_cells_ids, old_content_ids):
'''Write cells and contents to db'''
if old_cells_ids:
self._store.delete_published_cells(old_cells_ids)
if old_content_ids:
self._store.delete_published_contents(old_content_ids)
if cells:
self._store.create_published_cells(cells)
if contents:
self._store.create_published_contents(contents)
# @mongo_update_if_current_safe_retry
# def __update_user_if_current(self, user):
def _set_cell_roots(self, block, publish_request):
'''Set cell root'''
# Ensure here root assignment
old_ids = {}
deltas = block.deltas
last_time = len(deltas) - 2
for res in publish_request.cells:
old_name = publish_request.renames.get_old_name(res.name.cell_name)
old_id = block.cells.get_id(old_name, last_time)
if old_id:
old_ids[old_id] = res
else:
res.root = res.ID
old_cells = self._store.read_published_cells(old_ids.keys())
for old_id, old_cell in old_cells.iteritems():
res = old_ids[old_id]
res.root = old_cell.root
def _in_memory_block_update(self, block, publish_request):
'''Updates block in memory'''
self.security.check_write_block(block.ID)
cells, contents, old_cells_ids, old_content_ids = block.add_publication(publish_request,
self.auth_user)
self._set_cell_roots(block, publish_request)
return cells, contents, old_cells_ids, old_content_ids
def get_block_info(self, brl_block):
'''Check if auth_user can publish a block version specified by parameter block_version
Returns:
BlockInfo
'''
try:
self.security.check_read_block(brl_block)
except NotInStoreException:
# In this case, the block doesnt exist, but return information of -1 and permissions
return self._get_new_block_info(brl_block)
block_info = BlockInfo()
try:
self.security.check_write_block(brl_block)
block_info.can_write = True
except ForbiddenException:
block_info.can_write = False
try:
block = self._store.read_block(brl_block)
block_info.last_version = block.last_version()
block_info.private = self.security.is_private(brl_block)
except Exception as e:
tb = traceback.format_exc()
logger.debug(tb)
logger.error("Something went wrong with %s" % e)
raise BiiServiceException('Something went wrong')
return block_info
def _get_new_block_info(self, brl_block):
'''
Returns BlockInfo that new block would have if we publish it.
Raises exception if block cannot be created for any reason
'''
last_version = BlockVersion(brl_block, -1)
can_write = False
try:
self.security.check_create_block(brl_block.owner)
can_write = True
except ForbiddenException:
can_write = False
except NotInStoreException:
raise NotFoundException("Block %s not found!" % brl_block.to_pretty())
return BlockInfo(can_write=can_write, last_version=last_version)
def test_publish(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
# moduleID=BlockID(UserID(123),456)
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(
return_value=ElementPermissions(brl, private=False))
user = User("owner")
user.numeric_id = 1
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.last_version.return_value = Mock(BlockVersion)
block.add_publication.return_value = (Mock(list), Mock(list),
Mock(list), Mock(list))
block.deltas = []
ensure = Security('authUser', store)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_read_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r2.h'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r3.h'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
p.publish(pack)
block.add_publication.assert_called_once_with(pack, p.auth_user)
store.update_block.assert_called_once_with(block)
self.assertEqual(1, store.create_published_cells.call_count)
self.assertEqual(1, store.create_published_contents.call_count)
# Check sizes
self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3
# Publish again, see the size incremented
pack._bytes = None # Lazy computed
p.publish(pack)
self.assertEquals(user.blocks_bytes,
24) # 24 bytes: "hola" * 3 * 2 publications
# Max size exceeded for user
user.max_workspace_size = 25
self.assertRaises(ForbiddenException, p.publish, pack)
# Try to publish only 1 byte
pack._bytes = None # Lazy computed
pack.cells = []
pack.contents = {}
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('h'))
p.publish(pack)
def test_read_block(self):
# 1. Check we can always access a public block
brl = BRLBlock("goku/goku/block/master")
bper = ElementPermissions(brl, private=False)
self.store.upsert_block_permissions(bper)
ensure = Security("freezer", self.store)
self._subscribe("freezer", "enterprise_275_50_x")
ensure.check_read_block(brl)
# Even owner is not paying
self._subscribe("freezer", "free")
ensure.check_read_block(brl)
self._subscribe("freezer", "enterprise_275_50_x")
# 2. Check we can read a private block due to being the owner
bper = ElementPermissions(brl, private=True)
self.store.upsert_block_permissions(bper)
ensure = Security("goku", self.store)
self._subscribe("freezer", "enterprise_275_50_x")
ensure.check_read_block(brl)
# 4. Check we can read the block even subscription is not valid
self._subscribe("freezer", "free")
ensure.check_read_block(brl)
# 5. A user without read permissions cant read the block
ensure = Security("bulma", self.store)
self._subscribe("bulma", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_read_block, brl)
# 6. Until is granted as administrator or read granted
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_read_block(brl)
goku.administrators.revoke("bulma")
self.store.update_user(goku)
self.assertRaises(ForbiddenException, ensure.check_read_block, brl)
bper = ElementPermissions(brl, private=True)
bper.read.grant("bulma")
self.store.upsert_block_permissions(bper)
ensure.check_read_block(brl)
bper.read.remove("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_read_block, brl)
def __init__(self, store, auth_user):
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
self.translator = ReferenceTranslatorService(self._store,
self._auth_user)
class FindService(object):
MAX_HYP = 10
def __init__(self, store, auth_user):
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
self.translator = ReferenceTranslatorService(self._store,
self._auth_user)
def find(self, request, biiout):
'''
Params:
request: FinderRequest
biiout: biiout
Rerturns: FinderResult
'''
if not request:
raise ValueError('The find request is empty, nothing to find')
logger.debug('---------FinderRequest ------------\n%s' % str(request))
result = FinderResult()
# Copy unresolved and remove it if find the dependence
result.unresolved = copy(request.unresolved)
hypothesis = self._get_hypothesis(request, biiout)
if not hypothesis:
biiout.info("No block candidates found")
return result
biiout.info("Analyzing compatibility for found dependencies... ")
'''# primitive combinator variant
analyzer = CompatibilityAnalyzer(self._store, self._auth_user)
analysis_result = analyzer.solve(hypothesis)
# standard constraint variant
csp = CSPExact(hypothesis, None)
csp.solveCSP()
analysis_result = csp.getCompatibleSol()
logger.info(csp.print_info())'''
# iterative deepening variant
it = IterDeep(hypothesis, None, None)
sol_found, analysis_result = it.start()
if sol_found:
logger.info("sol found: {0} iter".format(it.num_iter))
if analysis_result is None:
biiout.error("Can't find a compatible solution")
return result
self._update_result(analysis_result, request, result, biiout)
if not result.unresolved:
if result.resolved:
biiout.info('All dependencies resolved')
elif not result.updated:
biiout.info('Everything was up to date')
logger.debug('Result %s' % result)
return result
def _get_hypothesis(self, request, biiresponse):
hypothesis = []
if request.find:
# group unresolved declarations by module
possible_blocks = request.possible_blocks()
logger.debug('Possible blocks %s' % possible_blocks)
for block_name, decls in possible_blocks.items():
hyp = self._compute_new(block_name, decls, request.policy,
request.block_names, biiresponse)
if len(hyp) > 0: # Don't append []
hypothesis.append(hyp)
existing_hypothesis = self._compute_existing(request, biiresponse)
if len(existing_hypothesis) > 0:
hypothesis.extend(existing_hypothesis)
logger.debug('Hypothesis %s' % hypothesis)
return hypothesis
def _update_result(self, analysis_result, request, result, biiout):
#existing = {version.block: version.time for version in request.existing}
for elem in analysis_result:
version = elem.block_version
for declaration, refs in elem.dep_dict.iteritems():
if declaration in request.unresolved:
biiout.debug("Resolved declaration %s" % str(declaration))
result.resolved[version][declaration] = refs
elif version not in request.existing:
biiout.info("Block %s updated to version %s" %
(version.block, version.time))
result.updated[version][declaration] = refs
# Remove cells from finder response unresolved
result.unresolved.difference_update(elem.dep_dict.keys())
def _compute_existing(self, request, biiout):
'''return a list of list of hypothesis for already defined (existing)
dependencies'''
result = []
for block_version, deps in request.existing.iteritems():
if request.update or request.downgrade or request.modify:
hypothesis = self._compute_modify(block_version, deps, request,
biiout)
else:
hypothesis = []
hypothesis.append(
Hypothesis(block_version, request.existing[block_version],
self.translator, request.block_names, biiout))
result.append(hypothesis)
return result
def _compute_modify(self, block_version, dependencies, request, biiout):
'''
Params:
block_version: Version to which dependencies are currently resolved to
dependencies: {Declaration: set(BlockCellName)}
request: FinderRequest
'''
brl_block = block_version.block
time = block_version.time
# First, compute all block candidates that have to be considered
block_candidates = {brl_block}
# Remove those not wanted by our policy
policy = request.policy
block_candidates = policy.filter(block_candidates)
current_block = self._store.read_block(brl_block)
original_date = current_block.deltas[time].date
delta_versions = self._filter_by_policy(block_candidates, policy,
biiout, original_date, request)
logger.debug("The heap is %s" % delta_versions)
hypothesis = self._define_hypothesis(delta_versions, dependencies,
request.block_names, biiout,
block_version)
return hypothesis
def _match_declarations(self, decls, block, snap, cur_version, version):
'''
Params:
decls: Current declarations for given block
block: Block to match
snap: dict {CellName => ID} for new version
cur_version: Current BlockVersion that decls are resolved to
version: New BlockVersion to evaluate
Return:
all_found: boolean
names_total: set(BlockCellName)
deps_dict: Dict {Declaration => set(BlockCellName)}
'''
all_found = True
deps_dict = {}
names_total = set()
block_name = block.ID.block_name
for decl in decls:
matchable_names = snap.keys()
renames = {}
if cur_version:
renames = block.get_renames(cur_version.time, version.time)
matchable_names.extend(
renames.keys()) # Adding old names so decls matches
names = decl.match([block_name + name for name in matchable_names])
names = set([
n if n.cell_name in snap.keys() else
(block_name + renames[n.cell_name]) for n in names
]) # Apply renames
if names:
# In case of renames here we will have a mismatch between declaration and cell_name
# it will be corrected by client by updating declaration when it detects such
# mismatch
deps_dict[decl] = names
names_total.update(names)
else:
all_found = False
break
return all_found, names_total, deps_dict
def _define_hypothesis(self,
delta_versions,
decls,
existing_block_names,
biiresponse,
cur_version=None):
'''
Parameters:
delta_versions: [(delta, block_version)], prioritized set of accepted hypothesis
decls: {Declaration: set(BlockCellName)}
existing_block_names = set(BlockName)
cur_version: Current version that decls are resolved to
Returns: list of hypothesis that match the required decls
'''
result = []
#repeated = set()
#previous = None
for _, version in delta_versions:
logger.debug('Analyzing hypothesis %s' % str(version))
block = self._store.read_block(version.block)
snap = block.cells.get_all_ids(version.time)
#logger.debug('Current snap %s' % snap)
all_found, names_total, deps_dict = self._match_declarations(
decls, block, snap, cur_version, version)
if not all_found:
biiresponse.debug(
'Version %s discarded, only contains files for declarations %s'
% (str(version), deps_dict.keys()))
continue
# Store the current IDs and dep table
#snap_contents = block.contents.get_ids(version.time)
#cell_ids = {snap[k.cell_name] for k in names_total}
#content_ids = {snap_contents[k.cell_name] for k in names_total if
# k.cell_name in snap_contents}
#dep_table = block.dep_tables.floor(version.time)
#current = cell_ids, content_ids, dep_table
# Only if the current option is different to the previous one
# we dont want to check the same option twice
#if previous != current and deps_dict:
logger.debug('Building hypothesis for %s with %s' %
(version, deps_dict))
# logger.debug('ref_dict %s' % ref_dict)
hyp = Hypothesis(version, deps_dict, self.translator,
existing_block_names, biiresponse)
result.append(hyp)
#previous = current
# FIXME: now the limit of hypothesis is hardwired
if len(result) >= FindService.MAX_HYP:
break
return result
def _filter_by_policy(self,
block_candidates,
policy,
biiresponse,
original_date=None,
request=None):
'''computes list of (block_delta, block_version) for each block candidate'''
delta_versions = []
for block_candidate in block_candidates:
self.security.check_read_block(block_candidate)
biiresponse.info("Block candidate: %s" % str(block_candidate))
block = self._store.read_block(block_candidate)
# from last to 0, backwards
for num_version in range(len(block.deltas) - 1, -1, -1):
tag = block.deltas[num_version].tag
date = block.deltas[num_version].date
if request:
if not request.downgrade and date <= original_date:
continue
version = BlockVersion(block.ID, num_version)
ev = policy.evaluate(version, tag)
if ev:
heappush(delta_versions, ((-date, -num_version), version))
biiresponse.info("\tVersion %s (%s) valid" %
(version, tag))
else:
biiresponse.info("\tVersion %s (%s) discarded" %
(version, tag))
return delta_versions
def _compute_new(self, block_name, decls, policy, existing_block_names,
biiresponse):
try:
biiresponse.info("Looking for %s..." % block_name)
# branches = self._store.read_tracks(block_name)
# branches.get_blocks()
block_candidates = [
block_name + BranchName("%s/master" % block_name.user)
]
block_candidates = policy.filter(block_candidates)
delta_versions = self._filter_by_policy(block_candidates, policy,
biiresponse)
logger.debug("The heap is %s" % delta_versions)
result = self._define_hypothesis(delta_versions, decls,
existing_block_names, biiresponse)
return result
except ForbiddenException: # Propagate forbidden to client
raise
except NotInStoreException:
biiresponse.warn("Can't find block candidate for: %s" %
(str(block_name)))
return []
except Exception:
biiresponse.error("Fatal error in server while reading %s" %
block_name)
logger.error(traceback.format_exc())
return []
def test_check_publish_block(self):
# 1. Onwer can write the block if its private
brl = BRLBlock("goku/goku/block/master")
self._add_block_to_user("goku", brl, True)
ensure = Security("goku", self.store)
self._subscribe("goku", "enterprise_275_50_x")
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.versiontag = 'mytag'
ensure.check_publish_block(brl, pack)
# If the owner is not paying he can't write
ensure = Security("goku", self.store)
self._subscribe("goku", "free")
self.assertRaises(ForbiddenException, ensure.check_publish_block, brl, pack)
self._subscribe("goku", "enterprise_275_50_x")
# 1b. But other not granted user can't write
ensure = Security("freezer", self.store)
self._subscribe("freezer", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_publish_block, brl, pack)
# 2. If bulma is granted as administrator or write he can write
ensure = Security("bulma", self.store)
self._subscribe("bulma", "enterprise_275_50_x")
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_publish_block(brl, pack)
goku.administrators.revoke("bulma")
self.store.update_user(goku)
self.assertRaises(ForbiddenException, ensure.check_publish_block, brl, pack)
bper = ElementPermissions(brl, private=True)
bper.write.grant("bulma")
self.store.upsert_block_permissions(bper)
ensure.check_publish_block(brl, pack)
bper.write.remove("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_publish_block, brl, pack)
# 3. If we give read permissions only, user cant write
bper = ElementPermissions(brl, private=True)
bper.read.grant("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_publish_block, brl, pack)
def get_renames(self, brl_block, t1, t2):
'''Gets 2 BlockVersion ([0],[1]) in a list and returns the renames'''
security = Security(self._auth_user, self._store)
security.check_read_block(brl_block)
block = self._store.read_block(brl_block)
return block.get_renames(t1, t2)
def test_write_block(self):
# 1. Onwer can write the block if its private
brl = BRLBlock("goku/goku/block/master")
self._add_block_to_user("goku", brl, True)
ensure = Security("goku", self.store)
self._subscribe("goku", "enterprise_275_50_x")
ensure.check_write_block(brl)
# If the owner is not paying he can't write
ensure = Security("goku", self.store)
self._subscribe("goku", "free")
self.assertRaises(ForbiddenException, ensure.check_write_block, brl)
self._subscribe("goku", "enterprise_275_50_x")
# 1b. But other not granted user can't write
ensure = Security("freezer", self.store)
self._subscribe("freezer", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_write_block, brl)
# 2. If bulma is granted as administrator or write he can write
ensure = Security("bulma", self.store)
self._subscribe("bulma", "enterprise_275_50_x")
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_write_block(brl)
goku.administrators.revoke("bulma")
self.store.update_user(goku)
self.assertRaises(ForbiddenException, ensure.check_write_block, brl)
bper = ElementPermissions(brl, private=True)
bper.write.grant("bulma")
self.store.upsert_block_permissions(bper)
ensure.check_write_block(brl)
bper.write.remove("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_write_block, brl)
# 3. If we give read permissions only, user cant write
bper = ElementPermissions(brl, private=True)
bper.read.grant("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_write_block, brl)
def test_change_subscription(self):
# Create users for test
self.store.create_user(User("cooper"))
ensure = Security("cooper", self.store)
self._subscribe("cooper", "startup_35_5_x")
# Coop has a private block
private_brl = BRLBlock("cooper/cooper/private_block/master")
self._add_block_to_user("cooper", private_brl, True)
for tmp in xrange(4):
new_user = BRLUser("user%s" % tmp)
ensure.check_grant_read_or_write_permissions_to(new_user, private_brl)
bper = self.store.read_block_permissions(private_brl)
bper.write.grant(new_user)
self.store.upsert_block_permissions(bper)
# Try to downgrade to personal plan
with self.assertRaisesRegexp(ForbiddenException, "You are currently using 4 users, "
"reduce to 1 before plan downgrade"):
ensure.check_can_change_current_subscription("cooper", "personal_7_1_x")
# Remove collaborators to 1
for tmp in xrange(3):
new_user = BRLUser("user%s" % tmp)
ensure.check_grant_read_or_write_permissions_to(new_user, private_brl)
bper = self.store.read_block_permissions(private_brl)
bper.write.revoke(new_user)
self.store.upsert_block_permissions(bper)
# Try to downgrade to personal plan
ensure.check_can_change_current_subscription("cooper", "personal_7_1_x")
# Try to downgrade to free plan
with self.assertRaisesRegexp(ForbiddenException, "You are currently using 1 users, "
"reduce to 0 before plan downgrade"):
ensure.check_can_change_current_subscription("cooper", "free")
# Remove last collaborator
bper = self.store.read_block_permissions(private_brl)
bper.write.revoke("user3")
self.store.upsert_block_permissions(bper)
with self.assertRaisesRegexp(ForbiddenException, "You have 1 private blocks, "
"reduce it to 0 before plan downgrade"):
ensure.check_can_change_current_subscription("cooper", "free")
def test_delete_block(self):
# First create master block permissions. Public
brl = BRLBlock("goku/goku/block/master")
bper_master = ElementPermissions(brl, private=True)
self.store.upsert_block_permissions(bper_master)
# 1. Owner can always delete a private block, even if i am not paying
ensure = Security("goku", self.store)
self._subscribe("goku", "enterprise_275_50_x")
ensure.check_delete_block(brl)
self._subscribe("freezer", "free")
ensure.check_delete_block(brl)
# 2. Other user can't delete block
ensure = Security("freezer", self.store)
self._subscribe("freezer", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_delete_block, brl)
# 3. A user with read and write permissions cant delete
ensure = Security("bulma", self.store)
bper = ElementPermissions(brl, private=True)
bper.write.grant("bulma")
bper.read.grant("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_delete_block, brl)
# 3. But an admin user can delete it
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_delete_block(brl)
def test_subscription_limits(self):
# Create users for test
self.store.create_user(User("cooper"))
self.store.create_user(User("bob"))
self.store.create_user(User("lady_lug"))
# Coop has a private block and a public block
public_brl = BRLBlock("cooper/cooper/public_block/master")
private_brl = BRLBlock("cooper/cooper/private_block/master")
self._add_block_to_user("cooper", public_brl, False)
self._add_block_to_user("cooper", private_brl, True)
# 1 contributor and unlimited private blocks
self._subscribe("cooper", "personal_7_1_x")
# Add an administrator
ensure = Security("cooper", self.store)
ensure.check_grant_administrator_for("cooper", "bob")
cooper = self.store.read_user("cooper")
cooper.administrators.grant("bob")
self.store.update_user(cooper)
# Try to add another one, it must fail
self.assertRaises(PlanUpgradeNeeded,
ensure.check_grant_administrator_for,
"cooper", "lady_lug")
# Try to add write permissions to a public block. Its ok
ensure.check_grant_read_or_write_permissions_to("lady_lug", public_brl)
# Try to add write permissions to a private block, but a bob (already admin)
ensure.check_grant_read_or_write_permissions_to("bob", private_brl)
# Try to add write permissions to a private block. It must fail
self.assertRaises(PlanUpgradeNeeded,
ensure.check_grant_read_or_write_permissions_to,
"lady_lug", private_brl)
# Remove Adminsitrator, try to add write permissions in a private block. Its ok
cooper = self.store.read_user("cooper")
cooper.administrators.revoke("bob")
self.store.update_user(cooper)
ensure.check_grant_read_or_write_permissions_to("lady_lug", private_brl)
cooper.administrators.grant("lady_lug")
self.store.update_user(cooper)
# Subscribe to a bigger plan and check limits
self._subscribe("cooper", "startup_35_5_x")
# Add 4 more
for tmp in xrange(4):
new_user = BRLUser("user%s" % tmp)
ensure.check_grant_read_or_write_permissions_to(new_user, private_brl)
bper = self.store.read_block_permissions(private_brl)
bper.write.grant(new_user)
self.store.upsert_block_permissions(bper)
# The sixth must fail
self.assertRaises(PlanUpgradeNeeded,
ensure.check_grant_read_or_write_permissions_to,
"new_user", private_brl)
# unless user is already a contributor
ensure.check_grant_read_or_write_permissions_to("lady_lug", private_brl)
def test_anonymous_operations(self):
ensure = Security(None, self.store)
ensure.check_read_block(self.public_brl)
class FindService(object):
MAX_HYP = 10
def __init__(self, store, auth_user):
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
self.translator = ReferenceTranslatorService(self._store, self._auth_user)
def find(self, request, biiout):
'''
Params:
request: FinderRequest
biiout: biiout
Rerturns: FinderResult
'''
if not request:
raise ValueError('The find request is empty, nothing to find')
logger.debug('---------FinderRequest ------------\n%s' % str(request))
result = FinderResult()
# Copy unresolved and remove it if find the dependence
result.unresolved = copy(request.unresolved)
hypothesis = self._get_hypothesis(request, biiout)
if not hypothesis:
biiout.info("No block candidates found")
return result
biiout.info("Analyzing compatibility for found dependencies... ")
'''# primitive combinator variant
analyzer = CompatibilityAnalyzer(self._store, self._auth_user)
analysis_result = analyzer.solve(hypothesis)
# standard constraint variant
csp = CSPExact(hypothesis, None)
csp.solveCSP()
analysis_result = csp.getCompatibleSol()
logger.info(csp.print_info())'''
# iterative deepening variant
it = IterDeep(hypothesis, None, None)
sol_found, analysis_result = it.start()
if sol_found:
logger.info("sol found: {0} iter".format(it.num_iter))
if analysis_result is None:
biiout.error("Can't find a compatible solution")
return result
self._update_result(analysis_result, request, result, biiout)
if not result.unresolved:
if result.resolved:
biiout.info('All dependencies resolved')
elif not result.updated:
biiout.info('Everything was up to date')
logger.debug('Result %s' % result)
return result
def _get_hypothesis(self, request, biiresponse):
hypothesis = []
if request.find:
# group unresolved declarations by module
possible_blocks = request.possible_blocks()
logger.debug('Possible blocks %s' % possible_blocks)
for block_name, decls in possible_blocks.items():
hyp = self._compute_new(block_name, decls, request.policy,
request.block_names, biiresponse)
if len(hyp) > 0: # Don't append []
hypothesis.append(hyp)
existing_hypothesis = self._compute_existing(request, biiresponse)
if len(existing_hypothesis) > 0:
hypothesis.extend(existing_hypothesis)
logger.debug('Hypothesis %s' % hypothesis)
return hypothesis
def _update_result(self, analysis_result, request, result, biiout):
#existing = {version.block: version.time for version in request.existing}
for elem in analysis_result:
version = elem.block_version
for declaration, refs in elem.dep_dict.iteritems():
if declaration in request.unresolved:
biiout.debug("Resolved declaration %s" % str(declaration))
result.resolved[version][declaration] = refs
elif version not in request.existing:
biiout.info("Block %s updated to version %s"
% (version.block, version.time))
result.updated[version][declaration] = refs
# Remove cells from finder response unresolved
result.unresolved.difference_update(elem.dep_dict.keys())
def _compute_existing(self, request, biiout):
'''return a list of list of hypothesis for already defined (existing)
dependencies'''
result = []
for block_version, deps in request.existing.iteritems():
if request.update or request.downgrade or request.modify:
hypothesis = self._compute_modify(block_version, deps, request, biiout)
else:
hypothesis = []
hypothesis.append(Hypothesis(block_version, request.existing[block_version],
self.translator, request.block_names, biiout))
result.append(hypothesis)
return result
def _compute_modify(self, block_version, dependencies, request, biiout):
'''
Params:
block_version: Version to which dependencies are currently resolved to
dependencies: {Declaration: set(BlockCellName)}
request: FinderRequest
'''
brl_block = block_version.block
time = block_version.time
# First, compute all block candidates that have to be considered
block_candidates = {brl_block}
# Remove those not wanted by our policy
policy = request.policy
block_candidates = policy.filter(block_candidates)
current_block = self._store.read_block(brl_block)
original_date = current_block.deltas[time].date
delta_versions = self._filter_by_policy(block_candidates, policy, biiout,
original_date, request)
logger.debug("The heap is %s" % delta_versions)
hypothesis = self._define_hypothesis(delta_versions, dependencies, request.block_names,
biiout, block_version)
return hypothesis
def _match_declarations(self, decls, block, snap, cur_version, version):
'''
Params:
decls: Current declarations for given block
block: Block to match
snap: dict {CellName => ID} for new version
cur_version: Current BlockVersion that decls are resolved to
version: New BlockVersion to evaluate
Return:
all_found: boolean
names_total: set(BlockCellName)
deps_dict: Dict {Declaration => set(BlockCellName)}
'''
all_found = True
deps_dict = {}
names_total = set()
block_name = block.ID.block_name
for decl in decls:
matchable_names = snap.keys()
renames = {}
if cur_version:
renames = block.get_renames(cur_version.time, version.time)
matchable_names.extend(renames.keys()) # Adding old names so decls matches
names = decl.match([block_name + name for name in matchable_names])
names = set([n if n.cell_name in snap.keys()
else (block_name + renames[n.cell_name]) for n in names]) # Apply renames
if names:
# In case of renames here we will have a mismatch between declaration and cell_name
# it will be corrected by client by updating declaration when it detects such
# mismatch
deps_dict[decl] = names
names_total.update(names)
else:
all_found = False
break
return all_found, names_total, deps_dict
def _define_hypothesis(self, delta_versions, decls, existing_block_names, biiresponse,
cur_version=None):
'''
Parameters:
delta_versions: [(delta, block_version)], prioritized set of accepted hypothesis
decls: {Declaration: set(BlockCellName)}
existing_block_names = set(BlockName)
cur_version: Current version that decls are resolved to
Returns: list of hypothesis that match the required decls
'''
result = []
#repeated = set()
#previous = None
for _, version in delta_versions:
logger.debug('Analyzing hypothesis %s' % str(version))
block = self._store.read_block(version.block)
snap = block.cells.get_all_ids(version.time)
#logger.debug('Current snap %s' % snap)
all_found, names_total, deps_dict = self._match_declarations(decls, block, snap,
cur_version, version)
if not all_found:
biiresponse.debug('Version %s discarded, only contains files for declarations %s'
% (str(version), deps_dict.keys()))
continue
# Store the current IDs and dep table
#snap_contents = block.contents.get_ids(version.time)
#cell_ids = {snap[k.cell_name] for k in names_total}
#content_ids = {snap_contents[k.cell_name] for k in names_total if
# k.cell_name in snap_contents}
#dep_table = block.dep_tables.floor(version.time)
#current = cell_ids, content_ids, dep_table
# Only if the current option is different to the previous one
# we dont want to check the same option twice
#if previous != current and deps_dict:
logger.debug('Building hypothesis for %s with %s' % (version, deps_dict))
# logger.debug('ref_dict %s' % ref_dict)
hyp = Hypothesis(version, deps_dict, self.translator, existing_block_names,
biiresponse)
result.append(hyp)
#previous = current
# FIXME: now the limit of hypothesis is hardwired
if len(result) >= FindService.MAX_HYP:
break
return result
def _filter_by_policy(self, block_candidates, policy, biiresponse,
original_date=None, request=None):
'''computes list of (block_delta, block_version) for each block candidate'''
delta_versions = []
for block_candidate in block_candidates:
self.security.check_read_block(block_candidate)
biiresponse.info("Block candidate: %s" % str(block_candidate))
block = self._store.read_block(block_candidate)
# from last to 0, backwards
for num_version in range(len(block.deltas) - 1, -1, -1):
tag = block.deltas[num_version].tag
date = block.deltas[num_version].date
if request:
if not request.downgrade and date <= original_date:
continue
version = BlockVersion(block.ID, num_version)
ev = policy.evaluate(version, tag)
if ev:
heappush(delta_versions, ((-date, -num_version), version))
biiresponse.info("\tVersion %s (%s) valid" % (version, tag))
else:
biiresponse.info("\tVersion %s (%s) discarded" % (version, tag))
return delta_versions
def _compute_new(self, block_name, decls, policy, existing_block_names, biiresponse):
try:
biiresponse.info("Looking for %s..." % block_name)
# branches = self._store.read_tracks(block_name)
# branches.get_blocks()
block_candidates = [block_name + BranchName("%s/master" % block_name.user)]
block_candidates = policy.filter(block_candidates)
delta_versions = self._filter_by_policy(block_candidates, policy, biiresponse)
logger.debug("The heap is %s" % delta_versions)
result = self._define_hypothesis(delta_versions, decls,
existing_block_names, biiresponse)
return result
except ForbiddenException: # Propagate forbidden to client
raise
except NotInStoreException:
biiresponse.warn("Can't find block candidate for: %s" % (str(block_name)))
return []
except Exception:
biiresponse.error("Fatal error in server while reading %s" % block_name)
logger.error(traceback.format_exc())
return []
def __init__(self, store, auth_user):
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
self.translator = ReferenceTranslatorService(self._store, self._auth_user)
def __init__(self, store, auth_user):
self._store = store
self.auth_user = auth_user
self.security = Security(self.auth_user, self._store)
class BiiService(BiiAPI):
'''Realization of the BiiAPI, the main entry point for functionality for server remote calls'''
def __init__(self, store, auth_user):
'''param store: an instance of GenericServerStore, could be in memory for testing or
MongoStore.
param auth_user: the user invoking this service, must be prior authenticated by app'''
self._store = store
self._auth_user = auth_user
self.security = Security(self._auth_user, self._store)
def get_server_info(self):
''' Gets the server info'''
from biicode.server.conf import BII_DOWNLOAD_URL
from biicode.server.conf import BII_GREET_MSG
si = ServerInfo(message=BII_GREET_MSG)
si.last_compatible = BII_LAST_COMPATIBLE_CLIENT
si.download_url = BII_DOWNLOAD_URL
return si
def publish(self, publish_request):
''' Publish in bii server'''
p = PublishService(self._store, self._auth_user)
return p.publish(publish_request)
def get_block_info(self, brl_block):
''' Read the block and get a BlockInfo object'''
p = PublishService(self._store, self._auth_user)
return p.get_block_info(brl_block)
def get_version_delta_info(self, block_version):
""" Read the delta info of a given block version
Raises: NotFoundException if version does not exist or is incongruent
"""
assert block_version.time is not None
try:
# FIXME: Optimize, reading all block only for get last version
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
if block_version.time > -1:
return block.deltas[block_version.time]
else:
return None
except NotInStoreException:
raise NotFoundException("Block %s not found!" % block_version.block.to_pretty())
except IndexError:
raise NotFoundException("Block version %s not found!" % str(block_version))
def get_version_by_tag(self, brl_block, version_tag):
"""Given a BlockVersion that has a tag but not a time returns a complete BlockVersion"""
assert version_tag is not None
try:
self.security.check_read_block(brl_block)
block = self._store.read_block(brl_block)
for time, delta in reversed(list(enumerate(block.deltas))):
if delta.versiontag == version_tag:
return BlockVersion(brl_block, time, version_tag)
raise NotFoundException("Block version %s: @%s not found!"
% (brl_block.to_pretty(), version_tag))
except NotInStoreException:
raise NotFoundException("Block %s not found!" % str(brl_block))
def get_published_resources(self, reference_dict):
''' Get the resources by their brl'''
r = ReferenceTranslatorService(self._store, self._auth_user)
return r.get_published_resources(reference_dict)
def get_dep_table(self, block_version):
''' Get the dependence table for this block version'''
assert block_version.time is not None
try:
r = ReferenceTranslatorService(self._store, self._auth_user)
return r.get_dep_table(block_version)
except NotInStoreException:
raise NotFoundException("Block %s not found!" % str(block_version.block))
def get_cells_snapshot(self, block_version):
''' Gets all cell names for an specific block version '''
assert block_version.time is not None
brl_block = block_version.block
try:
self.security.check_read_block(brl_block) # Security first, always
block = self._store.read_block(brl_block)
if block_version.time > len(block.deltas) - 1:
raise NotFoundException("There is no published version %d of %s\n" %
(block_version.time, block_version.block))
tmp = block.cells.get_all_ids(block_version.time) # Dict {cell_name: ID}
return tmp.keys()
except NotInStoreException:
raise NotFoundException("Block version %s not found!\n" % str(block_version))
def find(self, finder_request, response):
''' Find remote dependences '''
store = MemServerStore(self._store)
f = FindService(store, self._auth_user)
return f.find(finder_request, response)
def compute_diff(self, base_version, other_version):
''' Compare two versions '''
assert base_version.time is not None
assert other_version.time is not None
try:
self.security.check_read_block(base_version.block)
except NotInStoreException:
raise NotFoundException("Block version %s not found!\n" % str(base_version.block))
try:
self.security.check_read_block(other_version.block)
except NotInStoreException:
raise NotFoundException("Block version %s not found!\n" % str(other_version.block))
#This biiService is only for published versions
return compare_remote_versions(self, base_version, other_version)
def get_renames(self, brl_block, t1, t2):
'''Gets 2 BlockVersion ([0],[1]) in a list and returns the renames'''
security = Security(self._auth_user, self._store)
security.check_read_block(brl_block)
block = self._store.read_block(brl_block)
return block.get_renames(t1, t2)
def require_auth(self):
"""Only for validating token
(Used in publish manager to ensure logged user before publishing)"""
if not self._auth_user:
raise ForbiddenException()
def authenticate(self, username, password):
""" Create a "profile" object (object to encrypt) and expiration time.
Then return the JWT token Expiration time as a UTC UNIX timestamp
(an int) or as a datetime"""
user_service = UserService(self._store, self._auth_user)
try:
_, token = user_service.authenticate(username, password)
return token
except NotActivatedUser:
raise BiiRequestErrorException("User account: %s is not confirmed. Check your "
"email account and follow the instructions" % username)
def test_create_block(self):
brl = BRLBlock("goku/goku/block/master")
# Always can make a block if its public
ensure = Security("goku", self.store)
self._subscribe("goku", "free")
ensure.check_create_block(brl.owner, private=False)
# Only can make a private block if subscription is ok
self.assertRaises(ForbiddenException, ensure.check_create_block, brl.owner, private=True)
self._subscribe("goku", "enterprise_275_50_x")
ensure.check_create_block(brl.owner, private=True)
# Other user can create a block in my namespace if its an admin, not write and read is enought
ensure = Security("bulma", self.store)
self._subscribe("bulma", "enterprise_275_50_x")
self.assertRaises(ForbiddenException, ensure.check_create_block, brl.owner, private=True)
bper = ElementPermissions(brl.owner, private=True)
bper.write.grant("bulma")
bper.read.grant("bulma")
self.store.upsert_block_permissions(bper)
self.assertRaises(ForbiddenException, ensure.check_create_block, brl.owner, private=True)
# 3. But an admin user can delete it
goku = self.store.read_user("goku")
goku.administrators.grant("bulma")
self.store.update_user(goku)
ensure.check_create_block(brl.owner, private=True)
ensure.check_create_block(brl.owner, private=False)
class UserService(object):
"""Handles the registration, user profile updating, user confirmation.
"""
def __init__(self, store, auth_user):
self.store = store
self.auth_user = auth_user
self.security = Security(auth_user, store)
def edit_user(self, brl_user):
"""Get User fields for edition"""
self.security.check_update_user(brl_user)
user = self.get_user(brl_user)
user = user_to_json(user)
return user
def view_user(self, brl_user):
try:
user = self.get_user(brl_user)
except NotInStoreException:
raise NotFoundException("No user found with name %s" % brl_user)
# FIXME: Can read email
user_json = user_to_json(user)
del user_json["visible_email"]
del user_json["allow_mailing"]
if not user.visible_email and brl_user != self.auth_user:
user_json["email"] = None
return user_json
def get_user(self, brl_user):
'''Retrieve user information'''
try:
user = self.store.read_user(brl_user)
except NotInStoreException:
raise NotFoundException()
if not user.active:
raise NotFoundException()
# Not propagate sensible information
user.staff = None
user.last_api_call = None
user.active = None
user.confirmation_token = None
user.joined_date = None
user.confirmation_date = None
auth_blocks = {}
# Read all blocks and filter private ones
for brl_block, block_meta in user.blocks.iteritems():
try:
block_access = self.store.read_block_permissions(brl_block)
self.security.check_read_block(brl_block)
# block_meta => ([tags], description, bytes)
block_meta.append(block_access.is_private)
auth_blocks[brl_block] = block_meta
except ForbiddenException:
pass
user.blocks = auth_blocks
return user
def register(self, brl_user, email, plain_password, allow_mailing,
provider=None, access_token=None, invited_by=None):
'''
:param: user is a web_api.model.User
'''
# Validate password
if len(plain_password) < MIN_PASSWORD_LENGTH:
logger.debug("Invalid password length for %s" % email)
raise ControledErrorException("Password length must"
" be %s characters min" % MIN_PASSWORD_LENGTH)
# Search users with same email
if self.store.read_user_by_email(email):
logger.debug("Email '%s' already exists!" % email)
raise ControledErrorException("Email '%s' already exists! Forgot password? "
"Go to login and click on forgot password" % email)
try:
brl_user = BRLUser(brl_user)
bii_user = User(brl_user)
bii_user.password = plain_password
except InvalidNameException as e:
raise ControledErrorException(e)
# Search invited_by user (by mail or login)
friend = None
if invited_by:
if "@" in invited_by: # email address
friend = self.store.read_user_by_email(invited_by)
friend = friend.ID if friend else None
else: # Login
friend_object = self.store.exists_user_id_ignoring_case(invited_by)
if friend_object and friend_object.active:
friend = invited_by
if not friend:
raise ControledErrorException("User %s doesn't exist" % invited_by)
bii_user.invited_by = friend
# Check the existing of user name (User.ID), with case-insensitive
if self.store.exists_user_id_ignoring_case(brl_user):
logger.debug("User name '%s' already exists!" % brl_user)
raise ControledErrorException("Username '%s' already exists! "
"Choose other username" % brl_user)
try:
bii_user.email = email
bii_user.allow_mailing = allow_mailing
manager = JWTConfirmEmailManagerFactory.new()
token = manager.get_token_for(brl_user)
bii_user.confirmation_token = token
bii_user.joined_date = datetime.datetime.now()
bii_user.active = False
oauth_service = get_oauth_service(self.store)
oauth_user_info = oauth_service.get_user_info(provider, access_token)
self.store.create_user(bii_user)
if oauth_user_info:
# If user has changed the oauth email, not confirm the account
if oauth_user_info[1] == bii_user.email:
bii_user.active = True
try:
register_signup(self.store, brl_user)
except Exception as exc:
logger.error("Can't register sign-up in background! %s" % str(exc))
bii_user.fill_user_oauth_token(provider, access_token)
self.store.update_user(bii_user)
return bii_user
except Exception as e:
logger.error("Error creating user at mongo: %s" % str(e))
logger.error(traceback.format_exc())
raise e
def confirm_account(self, confirmation_token):
'''
Confirms user in database
'''
try:
# Decode token
jwt_manager = JWTConfirmEmailManagerFactory.new()
brl_user = jwt_manager.get_confirmed_user(confirmation_token)
user = self.store.read_user(brl_user)
except NotInStoreException:
raise NotFoundException("User '%s' doesn't exist" % brl_user)
if user.confirmation_token == confirmation_token:
if not user.active: # Do not re-send things if already activated
try:
register_signup(self.store, brl_user)
except Exception as exc:
logger.error("Can't register sign-up in background! %s" % str(exc))
user.active = True
user.confirmation_date = datetime.datetime.now()
self.store.update_user(user)
jwt_auth_manager = JWTCredentialsManagerFactory.new(self.store)
token = jwt_auth_manager.get_token_for(brl_user)
return token, brl_user, user.ga_client_id
else:
raise NotFoundException("Invalid user or token")
def confirm_password_reset(self, confirmation_token):
'''
Confirms password change. User and password are inside the token
'''
try:
# Decode token
jwt_manager = JWTPasswordResetManagerFactory.new()
brl_user, plain_password = jwt_manager.get_user_and_password(confirmation_token)
user = self.store.read_user(brl_user)
except Exception:
raise NotFoundException("No user found with name %s" % brl_user)
# Update password
user.password = plain_password
user.active = True # If not active, activate now, email is validated
self.store.update_user(user)
# Generate an auth token to autologin user
jwt_auth_manager = JWTCredentialsManagerFactory.new(self.store)
token = jwt_auth_manager.get_token_for(brl_user)
return token, brl_user
def update(self, brl_user, new_fields):
try:
self.security.check_update_user(brl_user)
user = self.store.read_user(brl_user)
user.firstname = new_fields["firstname"]
user.lastname = new_fields["lastname"]
user.country = new_fields["country"]
user.description = new_fields["description"]
user.street_1 = new_fields["street_1"]
user.street_2 = new_fields["street_2"]
user.city = new_fields["city"]
user.postal_code = new_fields["postal_code"]
user.region = new_fields["region"]
user.tax_id = new_fields["tax_id"]
user.vat = new_fields["vat"]
# Tsgs is for internal use yet
# user.tags = set(new_fields["tags"])
user.visible_email = new_fields["visible_email"]
user.allow_mailing = new_fields["allow_mailing"]
self.store.update_user(user)
except NotInStoreException:
raise NotFoundException("No user found with name %s" % brl_user)
def change_password(self, brl_user, old_password, new_plain_password):
''' Changes the password for the specified user'''
logger.debug("Change password for user %s" % brl_user)
self.security.check_change_password(brl_user)
user = self.store.read_user(brl_user)
if user.valid_password(old_password):
logger.debug("old password ok")
try:
user.password = new_plain_password
except InvalidNameException as e:
raise ControledErrorException(e)
self.store.update_user(user)
logger.debug("Updated user!")
else:
raise ControledErrorException("Invalid password!")
def authenticate(self, brl_user, password):
""" Create a "profile" object (object to encrypt) and expiration time.
Then return the JWT token Expiration time as a UTC UNIX timestamp
(an int) or as a datetime"""
try:
brl_user = BRLUser(brl_user)
except InvalidNameException:
raise AuthenticationException("Wrong user or password")
self._check_password(brl_user, password)
manager = JWTCredentialsManagerFactory.new(self.store)
token = manager.get_token_for(brl_user)
return brl_user, token
def _check_password(self, nickname, password):
''' Check user brl_user/password '''
try:
user = self.store.read_user(nickname)
except Exception:
raise AuthenticationException("Wrong user or password")
if user.active:
if not user.valid_password(password):
raise AuthenticationException("Wrong user or password")
else:
raise NotActivatedUser("User email is not confirmed! "
"We have sent an email to your account")
class PublishService(object):
''' Service for publish blocks in server.'''
def __init__(self, store, auth_user):
self._store = store
self.auth_user = auth_user
self.security = Security(self.auth_user, self._store)
def publish(self, publish_request):
'''Performs a publication
TIP: If we add publish_request to transaction_definition we can easily have asynchronous
publications
private: Only for first publication
'''
from biicode.server.background.enqueuer import register_publish
if publish_request.tag == DEV:
if not publish_request:
raise BiiRequestErrorException(
'Up to date, nothing to publish')
if publish_request.versiontag is not None:
raise PublishException('A DEV version cannot have tag %s' %
publish_request.tag)
assert publish_request.deptable is not None
# by default it is public
# TODO: BLock creation is not handled in the transaction
target_version = publish_request.parent
user = self._store.read_user(target_version.block.owner)
# Look if user has the block already created, because the block
# can exist with -1 version if it has been created in web
if target_version.block not in user.blocks.keys():
try:
if target_version != publish_request.parent: # Branching
user = self.create_block(target_version.block,
publish_request.parent,
private=False)
else:
user = self.create_block(target_version.block,
private=False)
except DuplicateBlockException:
pass # Its ok, already created
target_block = target_version.block
self._store.requestBlockTransaction(target_block)
try:
# If we can't read the block, we can't know about his existence
self.security.check_read_block(target_block)
self.security.check_publish_block(target_block, publish_request)
# biiresponse.debug('Read block "%s"' % brl_block)
block = self._store.read_block(target_block)
(cells, contents, old_cells_ids,
old_content_ids) = self._in_memory_block_update(
block, publish_request)
except ForbiddenException:
self._store.finishBlockTransaction(target_block)
raise
except PublishException as e:
self._store.finishBlockTransaction(target_block)
raise ServerInternalErrorException(e.message)
except Exception as excp:
logger.error("Exception in publish service!!: %s " % str(excp))
tb = traceback.format_exc()
logger.error(tb)
self._store.finishBlockTransaction(target_block)
raise ServerInternalErrorException()
self._store.beginBlockTransaction(target_block, cells, contents)
try:
self._write_resources_to_db(cells, contents, old_cells_ids,
old_content_ids)
self._store.update_block(block)
self._store.commitBlockTransaction(target_block)
register_publish(self.auth_user, block.last_version())
self._store.finishBlockTransaction(target_block)
# Need to read user again, otherwise will raise MongoNotCurrentObjectException
# because of double update of same memory object
user = self._store.read_user(target_version.block.owner)
user.add_block_size_bytes(target_version.block,
publish_request.bytes)
# Save user (with block bytes updated)
self._store.update_user(user)
return block.last_version()
except Exception as excp:
tb = traceback.format_exc()
logger.debug(tb)
self._rollback_transaction(excp, target_block)
raise ServerInternalErrorException(
'Publish transaction failed. Please, retry')
def create_block(self, brl, private=False):
'''Creates a block in server due the brl and description'''
self.security.check_create_block(brl.owner, private)
user = self._store.read_user(brl.owner)
try:
block_id = user.add_block(brl) # should fail if existing
except DuplicateBlockException:
logger.debug('Block %s already existing, not creating it' % brl)
raise
block = Block(block_id, brl)
try: # FIXME: better upsert?
self._store.create_block(block, private) # should fail if existing
except AlreadyInStoreException:
pass
self._store.update_user(user) # raise exception if not current
return user
def _rollback_transaction(self, excp, brl_block):
'''rollback transaction for publish'''
logger.warning(str(excp) + '\nRolling back publish transaction')
self._store.rollBackBlockTransaction(brl_block)
self._store.finishBlockTransaction(brl_block)
def _write_resources_to_db(self, cells, contents, old_cells_ids,
old_content_ids):
'''Write cells and contents to db'''
if old_cells_ids:
self._store.delete_published_cells(old_cells_ids)
if old_content_ids:
self._store.delete_published_contents(old_content_ids)
if cells:
self._store.create_published_cells(cells)
if contents:
self._store.create_published_contents(contents)
# @mongo_update_if_current_safe_retry
# def __update_user_if_current(self, user):
def _set_cell_roots(self, block, publish_request):
'''Set cell root'''
# Ensure here root assignment
old_ids = {}
deltas = block.deltas
last_time = len(deltas) - 2
for res in publish_request.cells:
old_name = publish_request.renames.get_old_name(res.name.cell_name)
old_id = block.cells.get_id(old_name, last_time)
if old_id:
old_ids[old_id] = res
else:
res.root = res.ID
old_cells = self._store.read_published_cells(old_ids.keys())
for old_id, old_cell in old_cells.iteritems():
res = old_ids[old_id]
res.root = old_cell.root
def _in_memory_block_update(self, block, publish_request):
'''Updates block in memory'''
self.security.check_write_block(block.ID)
cells, contents, old_cells_ids, old_content_ids = block.add_publication(
publish_request, self.auth_user)
self._set_cell_roots(block, publish_request)
return cells, contents, old_cells_ids, old_content_ids
def get_block_info(self, brl_block):
'''Check if auth_user can publish a block version specified by parameter block_version
Returns:
BlockInfo
'''
try:
self.security.check_read_block(brl_block)
except NotInStoreException:
# In this case, the block doesnt exist, but return information of -1 and permissions
return self._get_new_block_info(brl_block)
block_info = BlockInfo()
try:
self.security.check_write_block(brl_block)
block_info.can_write = True
except ForbiddenException:
block_info.can_write = False
try:
block = self._store.read_block(brl_block)
block_info.last_version = block.last_version()
block_info.private = self.security.is_private(brl_block)
except Exception as e:
tb = traceback.format_exc()
logger.debug(tb)
logger.error("Something went wrong with %s" % e)
raise BiiServiceException('Something went wrong')
return block_info
def _get_new_block_info(self, brl_block):
'''
Returns BlockInfo that new block would have if we publish it.
Raises exception if block cannot be created for any reason
'''
last_version = BlockVersion(brl_block, -1)
can_write = False
try:
self.security.check_create_block(brl_block.owner)
can_write = True
except ForbiddenException:
can_write = False
except NotInStoreException:
raise NotFoundException("Block %s not found!" %
brl_block.to_pretty())
return BlockInfo(can_write=can_write, last_version=last_version)
def __init__(self, store, auth_user):
self._store = store
self.security = Security(auth_user, self._store)
class ReferenceTranslatorService(object):
def __init__(self, store, auth_user):
self._store = store
self.security = Security(auth_user, self._store)
def get_published_resources(self, references):
result = ReferencedResources()
for block_version, cell_names in references.iteritems():
try:
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
cell_ids = block.cells.get_ids(cell_names, block_version.time)
content_ids = block.contents.get_ids(cell_names,
block_version.time)
cells = self._store.read_published_cells(cell_ids.values())
contents = self._store.read_published_contents(
content_ids.values())
for name, rID in cell_ids.iteritems():
if name in content_ids:
cid = content_ids[name]
cid = contents[cid]
else:
cid = None # Virtual resource
result[block_version][name] = Resource(cells[rID], cid)
except (ForbiddenException, NotInStoreException):
pass
return result
def get_published_min_refs(self, references):
'''returns the minimum information required to perform a compatibility check for those
references. This method is currently used just by CompatibilityClosureBuilder
param references: {block_version: set(cell_names)}
return: {block_version: {cell_name: (cell_id, content_id), root_id, [deps blockcellnames]}}
'''
result = defaultdict(dict)
for block_version, cell_names in references.iteritems():
try:
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
cell_ids = block.cells.get_ids(cell_names, block_version.time)
content_ids = block.contents.get_ids(cell_names,
block_version.time)
cells = self._store.read_min_cells(cell_ids.values())
#This cells are {cellID: (rootID, dep_block_names)}
for cell_name, cell_id in cell_ids.iteritems():
content_id = content_ids.get(
cell_name) # None if Virtual resource
root_id, deps = cells.get(cell_id, (None, None))
if root_id is not None:
result[block_version][cell_name] = ((cell_id,
content_id),
root_id, deps)
except (ForbiddenException, NotInStoreException):
pass
return result
def get_dep_table(self, block_version):
self.security.check_read_block(block_version.block)
block = self._store.read_block(block_version.block)
table = block.dep_tables.find(block_version.time)
return table
