def test_publish_dev_with_tag(self, enqueuer): brl = BRLBlock('owner/user/block/branch') store = Mock(MongoServerStore) store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False)) user = Mock() user.blocks = {} store.read_user = Mock(return_value=user) block = Mock(Block) block.add_publication.return_value = (['mock_id'], [], [], []) block.deltas = [] ensure = Security('authUser', store) ensure.check_read_block = Mock(return_value=True) ensure.check_create_block = Mock(return_value=True) ensure.check_write_block = Mock(return_value=True) ensure.check_publish_block = Mock(return_value=True) store.read_block.return_value = block store.read_published_cells.return_value = {} p = PublishService(store, 'authUser') p.security = ensure pack = PublishRequest(BlockVersion(brl, -1)) pack.cells.append(SimpleCell('user/block/r1.h')) pack.deptable = BlockVersionTable() p.publish(pack)
def test_publish_dev_with_tag(self, enqueuer): brl = BRLBlock('owner/user/block/branch') store = Mock(MongoServerStore) store.read_block_permissions = Mock( return_value=ElementPermissions(brl, private=False)) user = Mock() user.blocks = {} store.read_user = Mock(return_value=user) block = Mock(Block) block.add_publication.return_value = (['mock_id'], [], [], []) block.deltas = [] ensure = Security('authUser', store) ensure.check_read_block = Mock(return_value=True) ensure.check_create_block = Mock(return_value=True) ensure.check_write_block = Mock(return_value=True) ensure.check_publish_block = Mock(return_value=True) store.read_block.return_value = block store.read_published_cells.return_value = {} p = PublishService(store, 'authUser') p.security = ensure pack = PublishRequest(BlockVersion(brl, -1)) pack.cells.append(SimpleCell('user/block/r1.h')) pack.deptable = BlockVersionTable() p.publish(pack)
def test_publish(self, enqueuer): brl = BRLBlock('owner/user/block/branch') # moduleID=BlockID(UserID(123),456) store = Mock(MongoServerStore) store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False)) user = User("owner") user.numeric_id = 1 user.blocks = {} store.read_user = Mock(return_value=user) block = Mock(Block) block.last_version.return_value = Mock(BlockVersion) block.add_publication.return_value = (Mock(list), Mock(list), Mock(list), Mock(list)) block.deltas = [] ensure = Security('authUser', store) ensure.check_create_block = Mock(return_value=True) ensure.check_write_block = Mock(return_value=True) ensure.check_read_block = Mock(return_value=True) store.read_block.return_value = block store.read_published_cells.return_value = {} p = PublishService(store, 'authUser') p.security = ensure pack = PublishRequest(BlockVersion(brl, -1)) pack.cells.append(SimpleCell('user/block/r1.h')) pack.contents['r1.h'] = Content(id_=None, load=Blob('hola')) pack.cells.append(SimpleCell('user/block/r2.h')) pack.contents['r2.h'] = Content(id_=None, load=Blob('hola')) pack.cells.append(SimpleCell('user/block/r3.h')) pack.contents['r3.h'] = Content(id_=None, load=Blob('hola')) pack.deptable = BlockVersionTable() p.publish(pack) block.add_publication.assert_called_once_with(pack, p.auth_user) store.update_block.assert_called_once_with(block) self.assertEqual(1, store.create_published_cells.call_count) self.assertEqual(1, store.create_published_contents.call_count) # Check sizes self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3 # Publish again, see the size incremented pack._bytes = None # Lazy computed p.publish(pack) self.assertEquals(user.blocks_bytes, 24) # 24 bytes: "hola" * 3 * 2 publications # Max size exceeded for user user.max_workspace_size = 25 self.assertRaises(ForbiddenException, p.publish, pack) # Try to publish only 1 byte pack._bytes = None # Lazy computed pack.cells = [] pack.contents = {} pack.cells.append(SimpleCell('user/block/r1.h')) pack.contents['r1.h'] = Content(id_=None, load=Blob('h')) p.publish(pack)
class ReferenceTranslatorService(object): def __init__(self, store, auth_user): self._store = store self.security = Security(auth_user, self._store) def get_published_resources(self, references): result = ReferencedResources() for block_version, cell_names in references.iteritems(): try: self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) cell_ids = block.cells.get_ids(cell_names, block_version.time) content_ids = block.contents.get_ids(cell_names, block_version.time) cells = self._store.read_published_cells(cell_ids.values()) contents = self._store.read_published_contents(content_ids.values()) for name, rID in cell_ids.iteritems(): if name in content_ids: cid = content_ids[name] cid = contents[cid] else: cid = None # Virtual resource result[block_version][name] = Resource(cells[rID], cid) except (ForbiddenException, NotInStoreException): pass return result def get_published_min_refs(self, references): """returns the minimum information required to perform a compatibility check for those references. This method is currently used just by CompatibilityClosureBuilder param references: {block_version: set(cell_names)} return: {block_version: {cell_name: (cell_id, content_id), root_id, [deps blockcellnames]}} """ result = defaultdict(dict) for block_version, cell_names in references.iteritems(): try: self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) cell_ids = block.cells.get_ids(cell_names, block_version.time) content_ids = block.contents.get_ids(cell_names, block_version.time) cells = self._store.read_min_cells(cell_ids.values()) # This cells are {cellID: (rootID, dep_block_names)} for cell_name, cell_id in cell_ids.iteritems(): content_id = content_ids.get(cell_name) # None if Virtual resource root_id, deps = cells.get(cell_id, (None, None)) if root_id is not None: result[block_version][cell_name] = ((cell_id, content_id), root_id, deps) except (ForbiddenException, NotInStoreException): pass return result def get_dep_table(self, block_version): self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) table = block.dep_tables.find(block_version.time) return table
class UserService(object): """Handles the registration, user profile updating, user confirmation. """ def __init__(self, store, auth_user): self.store = store self.auth_user = auth_user self.security = Security(auth_user, store) def edit_user(self, brl_user): """Get User fields for edition""" self.security.check_update_user(brl_user) user = self.get_user(brl_user) user = user_to_json(user) return user def view_user(self, brl_user): try: user = self.get_user(brl_user) except NotInStoreException: raise NotFoundException("No user found with name %s" % brl_user) # FIXME: Can read email user_json = user_to_json(user) del user_json["visible_email"] del user_json["allow_mailing"] if not user.visible_email and brl_user != self.auth_user: user_json["email"] = None return user_json def get_user(self, brl_user): '''Retrieve user information''' try: user = self.store.read_user(brl_user) except NotInStoreException: raise NotFoundException() if not user.active: raise NotFoundException() # Not propagate sensible information user.staff = None user.last_api_call = None user.active = None user.confirmation_token = None user.joined_date = None user.confirmation_date = None auth_blocks = {} # Read all blocks and filter private ones for brl_block, block_meta in user.blocks.iteritems(): try: block_access = self.store.read_block_permissions(brl_block) self.security.check_read_block(brl_block) # block_meta => ([tags], description, bytes) block_meta.append(block_access.is_private) auth_blocks[brl_block] = block_meta except ForbiddenException: pass user.blocks = auth_blocks return user def register(self, brl_user, email, plain_password, allow_mailing, provider=None, access_token=None, invited_by=None): ''' :param: user is a web_api.model.User ''' # Validate password if len(plain_password) < MIN_PASSWORD_LENGTH: logger.debug("Invalid password length for %s" % email) raise ControledErrorException("Password length must" " be %s characters min" % MIN_PASSWORD_LENGTH) # Search users with same email if self.store.read_user_by_email(email): logger.debug("Email '%s' already exists!" % email) raise ControledErrorException("Email '%s' already exists! Forgot password? " "Go to login and click on forgot password" % email) try: brl_user = BRLUser(brl_user) bii_user = User(brl_user) bii_user.password = plain_password except InvalidNameException as e: raise ControledErrorException(e) # Search invited_by user (by mail or login) friend = None if invited_by: if "@" in invited_by: # email address friend = self.store.read_user_by_email(invited_by) friend = friend.ID if friend else None else: # Login friend_object = self.store.exists_user_id_ignoring_case(invited_by) if friend_object and friend_object.active: friend = invited_by if not friend: raise ControledErrorException("User %s doesn't exist" % invited_by) bii_user.invited_by = friend # Check the existing of user name (User.ID), with case-insensitive if self.store.exists_user_id_ignoring_case(brl_user): logger.debug("User name '%s' already exists!" % brl_user) raise ControledErrorException("Username '%s' already exists! " "Choose other username" % brl_user) try: bii_user.email = email bii_user.allow_mailing = allow_mailing manager = JWTConfirmEmailManagerFactory.new() token = manager.get_token_for(brl_user) bii_user.confirmation_token = token bii_user.joined_date = datetime.datetime.now() bii_user.active = False oauth_service = get_oauth_service(self.store) oauth_user_info = oauth_service.get_user_info(provider, access_token) self.store.create_user(bii_user) if oauth_user_info: # If user has changed the oauth email, not confirm the account if oauth_user_info[1] == bii_user.email: bii_user.active = True try: register_signup(self.store, brl_user) except Exception as exc: logger.error("Can't register sign-up in background! %s" % str(exc)) bii_user.fill_user_oauth_token(provider, access_token) self.store.update_user(bii_user) return bii_user except Exception as e: logger.error("Error creating user at mongo: %s" % str(e)) logger.error(traceback.format_exc()) raise e def confirm_account(self, confirmation_token): ''' Confirms user in database ''' try: # Decode token jwt_manager = JWTConfirmEmailManagerFactory.new() brl_user = jwt_manager.get_confirmed_user(confirmation_token) user = self.store.read_user(brl_user) except NotInStoreException: raise NotFoundException("User '%s' doesn't exist" % brl_user) if user.confirmation_token == confirmation_token: if not user.active: # Do not re-send things if already activated try: register_signup(self.store, brl_user) except Exception as exc: logger.error("Can't register sign-up in background! %s" % str(exc)) user.active = True user.confirmation_date = datetime.datetime.now() self.store.update_user(user) jwt_auth_manager = JWTCredentialsManagerFactory.new(self.store) token = jwt_auth_manager.get_token_for(brl_user) return token, brl_user, user.ga_client_id else: raise NotFoundException("Invalid user or token") def confirm_password_reset(self, confirmation_token): ''' Confirms password change. User and password are inside the token ''' try: # Decode token jwt_manager = JWTPasswordResetManagerFactory.new() brl_user, plain_password = jwt_manager.get_user_and_password(confirmation_token) user = self.store.read_user(brl_user) except Exception: raise NotFoundException("No user found with name %s" % brl_user) # Update password user.password = plain_password user.active = True # If not active, activate now, email is validated self.store.update_user(user) # Generate an auth token to autologin user jwt_auth_manager = JWTCredentialsManagerFactory.new(self.store) token = jwt_auth_manager.get_token_for(brl_user) return token, brl_user def update(self, brl_user, new_fields): try: self.security.check_update_user(brl_user) user = self.store.read_user(brl_user) user.firstname = new_fields["firstname"] user.lastname = new_fields["lastname"] user.country = new_fields["country"] user.description = new_fields["description"] user.street_1 = new_fields["street_1"] user.street_2 = new_fields["street_2"] user.city = new_fields["city"] user.postal_code = new_fields["postal_code"] user.region = new_fields["region"] user.tax_id = new_fields["tax_id"] user.vat = new_fields["vat"] # Tsgs is for internal use yet # user.tags = set(new_fields["tags"]) user.visible_email = new_fields["visible_email"] user.allow_mailing = new_fields["allow_mailing"] self.store.update_user(user) except NotInStoreException: raise NotFoundException("No user found with name %s" % brl_user) def change_password(self, brl_user, old_password, new_plain_password): ''' Changes the password for the specified user''' logger.debug("Change password for user %s" % brl_user) self.security.check_change_password(brl_user) user = self.store.read_user(brl_user) if user.valid_password(old_password): logger.debug("old password ok") try: user.password = new_plain_password except InvalidNameException as e: raise ControledErrorException(e) self.store.update_user(user) logger.debug("Updated user!") else: raise ControledErrorException("Invalid password!") def authenticate(self, brl_user, password): """ Create a "profile" object (object to encrypt) and expiration time. Then return the JWT token Expiration time as a UTC UNIX timestamp (an int) or as a datetime""" try: brl_user = BRLUser(brl_user) except InvalidNameException: raise AuthenticationException("Wrong user or password") self._check_password(brl_user, password) manager = JWTCredentialsManagerFactory.new(self.store) token = manager.get_token_for(brl_user) return brl_user, token def _check_password(self, nickname, password): ''' Check user brl_user/password ''' try: user = self.store.read_user(nickname) except Exception: raise AuthenticationException("Wrong user or password") if user.active: if not user.valid_password(password): raise AuthenticationException("Wrong user or password") else: raise NotActivatedUser("User email is not confirmed! " "We have sent an email to your account")
class BiiService(BiiAPI): '''Realization of the BiiAPI, the main entry point for functionality for server remote calls''' def __init__(self, store, auth_user): '''param store: an instance of GenericServerStore, could be in memory for testing or MongoStore. param auth_user: the user invoking this service, must be prior authenticated by app''' self._store = store self._auth_user = auth_user self.security = Security(self._auth_user, self._store) def get_server_info(self): ''' Gets the server info''' from biicode.server.conf import BII_DOWNLOAD_URL from biicode.server.conf import BII_GREET_MSG si = ServerInfo(message=BII_GREET_MSG) si.last_compatible = BII_LAST_COMPATIBLE_CLIENT si.download_url = BII_DOWNLOAD_URL return si def publish(self, publish_request): ''' Publish in bii server''' p = PublishService(self._store, self._auth_user) return p.publish(publish_request) def get_block_info(self, brl_block): ''' Read the block and get a BlockInfo object''' p = PublishService(self._store, self._auth_user) return p.get_block_info(brl_block) def get_version_delta_info(self, block_version): """ Read the delta info of a given block version Raises: NotFoundException if version does not exist or is incongruent """ assert block_version.time is not None try: # FIXME: Optimize, reading all block only for get last version self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) if block_version.time > -1: return block.deltas[block_version.time] else: return None except NotInStoreException: raise NotFoundException("Block %s not found!" % block_version.block.to_pretty()) except IndexError: raise NotFoundException("Block version %s not found!" % str(block_version)) def get_version_by_tag(self, brl_block, version_tag): """Given a BlockVersion that has a tag but not a time returns a complete BlockVersion""" assert version_tag is not None try: self.security.check_read_block(brl_block) block = self._store.read_block(brl_block) for time, delta in reversed(list(enumerate(block.deltas))): if delta.versiontag == version_tag: return BlockVersion(brl_block, time, version_tag) raise NotFoundException("Block version %s: @%s not found!" % (brl_block.to_pretty(), version_tag)) except NotInStoreException: raise NotFoundException("Block %s not found!" % str(brl_block)) def get_published_resources(self, reference_dict): ''' Get the resources by their brl''' r = ReferenceTranslatorService(self._store, self._auth_user) return r.get_published_resources(reference_dict) def get_dep_table(self, block_version): ''' Get the dependence table for this block version''' assert block_version.time is not None try: r = ReferenceTranslatorService(self._store, self._auth_user) return r.get_dep_table(block_version) except NotInStoreException: raise NotFoundException("Block %s not found!" % str(block_version.block)) def get_cells_snapshot(self, block_version): ''' Gets all cell names for an specific block version ''' assert block_version.time is not None brl_block = block_version.block try: self.security.check_read_block(brl_block) # Security first, always block = self._store.read_block(brl_block) if block_version.time > len(block.deltas) - 1: raise NotFoundException("There is no published version %d of %s\n" % (block_version.time, block_version.block)) tmp = block.cells.get_all_ids(block_version.time) # Dict {cell_name: ID} return tmp.keys() except NotInStoreException: raise NotFoundException("Block version %s not found!\n" % str(block_version)) def find(self, finder_request, response): ''' Find remote dependences ''' store = MemServerStore(self._store) f = FindService(store, self._auth_user) return f.find(finder_request, response) def compute_diff(self, base_version, other_version): ''' Compare two versions ''' assert base_version.time is not None assert other_version.time is not None try: self.security.check_read_block(base_version.block) except NotInStoreException: raise NotFoundException("Block version %s not found!\n" % str(base_version.block)) try: self.security.check_read_block(other_version.block) except NotInStoreException: raise NotFoundException("Block version %s not found!\n" % str(other_version.block)) #This biiService is only for published versions return compare_remote_versions(self, base_version, other_version) def get_renames(self, brl_block, t1, t2): '''Gets 2 BlockVersion ([0],[1]) in a list and returns the renames''' security = Security(self._auth_user, self._store) security.check_read_block(brl_block) block = self._store.read_block(brl_block) return block.get_renames(t1, t2) def require_auth(self): """Only for validating token (Used in publish manager to ensure logged user before publishing)""" if not self._auth_user: raise ForbiddenException() def authenticate(self, username, password): """ Create a "profile" object (object to encrypt) and expiration time. Then return the JWT token Expiration time as a UTC UNIX timestamp (an int) or as a datetime""" user_service = UserService(self._store, self._auth_user) try: _, token = user_service.authenticate(username, password) return token except NotActivatedUser: raise BiiRequestErrorException("User account: %s is not confirmed. Check your " "email account and follow the instructions" % username)
def get_renames(self, brl_block, t1, t2): '''Gets 2 BlockVersion ([0],[1]) in a list and returns the renames''' security = Security(self._auth_user, self._store) security.check_read_block(brl_block) block = self._store.read_block(brl_block) return block.get_renames(t1, t2)
class FindService(object): MAX_HYP = 10 def __init__(self, store, auth_user): self._store = store self._auth_user = auth_user self.security = Security(self._auth_user, self._store) self.translator = ReferenceTranslatorService(self._store, self._auth_user) def find(self, request, biiout): ''' Params: request: FinderRequest biiout: biiout Rerturns: FinderResult ''' if not request: raise ValueError('The find request is empty, nothing to find') logger.debug('---------FinderRequest ------------\n%s' % str(request)) result = FinderResult() # Copy unresolved and remove it if find the dependence result.unresolved = copy(request.unresolved) hypothesis = self._get_hypothesis(request, biiout) if not hypothesis: biiout.info("No block candidates found") return result biiout.info("Analyzing compatibility for found dependencies... ") '''# primitive combinator variant analyzer = CompatibilityAnalyzer(self._store, self._auth_user) analysis_result = analyzer.solve(hypothesis) # standard constraint variant csp = CSPExact(hypothesis, None) csp.solveCSP() analysis_result = csp.getCompatibleSol() logger.info(csp.print_info())''' # iterative deepening variant it = IterDeep(hypothesis, None, None) sol_found, analysis_result = it.start() if sol_found: logger.info("sol found: {0} iter".format(it.num_iter)) if analysis_result is None: biiout.error("Can't find a compatible solution") return result self._update_result(analysis_result, request, result, biiout) if not result.unresolved: if result.resolved: biiout.info('All dependencies resolved') elif not result.updated: biiout.info('Everything was up to date') logger.debug('Result %s' % result) return result def _get_hypothesis(self, request, biiresponse): hypothesis = [] if request.find: # group unresolved declarations by module possible_blocks = request.possible_blocks() logger.debug('Possible blocks %s' % possible_blocks) for block_name, decls in possible_blocks.items(): hyp = self._compute_new(block_name, decls, request.policy, request.block_names, biiresponse) if len(hyp) > 0: # Don't append [] hypothesis.append(hyp) existing_hypothesis = self._compute_existing(request, biiresponse) if len(existing_hypothesis) > 0: hypothesis.extend(existing_hypothesis) logger.debug('Hypothesis %s' % hypothesis) return hypothesis def _update_result(self, analysis_result, request, result, biiout): #existing = {version.block: version.time for version in request.existing} for elem in analysis_result: version = elem.block_version for declaration, refs in elem.dep_dict.iteritems(): if declaration in request.unresolved: biiout.debug("Resolved declaration %s" % str(declaration)) result.resolved[version][declaration] = refs elif version not in request.existing: biiout.info("Block %s updated to version %s" % (version.block, version.time)) result.updated[version][declaration] = refs # Remove cells from finder response unresolved result.unresolved.difference_update(elem.dep_dict.keys()) def _compute_existing(self, request, biiout): '''return a list of list of hypothesis for already defined (existing) dependencies''' result = [] for block_version, deps in request.existing.iteritems(): if request.update or request.downgrade or request.modify: hypothesis = self._compute_modify(block_version, deps, request, biiout) else: hypothesis = [] hypothesis.append(Hypothesis(block_version, request.existing[block_version], self.translator, request.block_names, biiout)) result.append(hypothesis) return result def _compute_modify(self, block_version, dependencies, request, biiout): ''' Params: block_version: Version to which dependencies are currently resolved to dependencies: {Declaration: set(BlockCellName)} request: FinderRequest ''' brl_block = block_version.block time = block_version.time # First, compute all block candidates that have to be considered block_candidates = {brl_block} # Remove those not wanted by our policy policy = request.policy block_candidates = policy.filter(block_candidates) current_block = self._store.read_block(brl_block) original_date = current_block.deltas[time].date delta_versions = self._filter_by_policy(block_candidates, policy, biiout, original_date, request) logger.debug("The heap is %s" % delta_versions) hypothesis = self._define_hypothesis(delta_versions, dependencies, request.block_names, biiout, block_version) return hypothesis def _match_declarations(self, decls, block, snap, cur_version, version): ''' Params: decls: Current declarations for given block block: Block to match snap: dict {CellName => ID} for new version cur_version: Current BlockVersion that decls are resolved to version: New BlockVersion to evaluate Return: all_found: boolean names_total: set(BlockCellName) deps_dict: Dict {Declaration => set(BlockCellName)} ''' all_found = True deps_dict = {} names_total = set() block_name = block.ID.block_name for decl in decls: matchable_names = snap.keys() renames = {} if cur_version: renames = block.get_renames(cur_version.time, version.time) matchable_names.extend(renames.keys()) # Adding old names so decls matches names = decl.match([block_name + name for name in matchable_names]) names = set([n if n.cell_name in snap.keys() else (block_name + renames[n.cell_name]) for n in names]) # Apply renames if names: # In case of renames here we will have a mismatch between declaration and cell_name # it will be corrected by client by updating declaration when it detects such # mismatch deps_dict[decl] = names names_total.update(names) else: all_found = False break return all_found, names_total, deps_dict def _define_hypothesis(self, delta_versions, decls, existing_block_names, biiresponse, cur_version=None): ''' Parameters: delta_versions: [(delta, block_version)], prioritized set of accepted hypothesis decls: {Declaration: set(BlockCellName)} existing_block_names = set(BlockName) cur_version: Current version that decls are resolved to Returns: list of hypothesis that match the required decls ''' result = [] #repeated = set() #previous = None for _, version in delta_versions: logger.debug('Analyzing hypothesis %s' % str(version)) block = self._store.read_block(version.block) snap = block.cells.get_all_ids(version.time) #logger.debug('Current snap %s' % snap) all_found, names_total, deps_dict = self._match_declarations(decls, block, snap, cur_version, version) if not all_found: biiresponse.debug('Version %s discarded, only contains files for declarations %s' % (str(version), deps_dict.keys())) continue # Store the current IDs and dep table #snap_contents = block.contents.get_ids(version.time) #cell_ids = {snap[k.cell_name] for k in names_total} #content_ids = {snap_contents[k.cell_name] for k in names_total if # k.cell_name in snap_contents} #dep_table = block.dep_tables.floor(version.time) #current = cell_ids, content_ids, dep_table # Only if the current option is different to the previous one # we dont want to check the same option twice #if previous != current and deps_dict: logger.debug('Building hypothesis for %s with %s' % (version, deps_dict)) # logger.debug('ref_dict %s' % ref_dict) hyp = Hypothesis(version, deps_dict, self.translator, existing_block_names, biiresponse) result.append(hyp) #previous = current # FIXME: now the limit of hypothesis is hardwired if len(result) >= FindService.MAX_HYP: break return result def _filter_by_policy(self, block_candidates, policy, biiresponse, original_date=None, request=None): '''computes list of (block_delta, block_version) for each block candidate''' delta_versions = [] for block_candidate in block_candidates: self.security.check_read_block(block_candidate) biiresponse.info("Block candidate: %s" % str(block_candidate)) block = self._store.read_block(block_candidate) # from last to 0, backwards for num_version in range(len(block.deltas) - 1, -1, -1): tag = block.deltas[num_version].tag date = block.deltas[num_version].date if request: if not request.downgrade and date <= original_date: continue version = BlockVersion(block.ID, num_version) ev = policy.evaluate(version, tag) if ev: heappush(delta_versions, ((-date, -num_version), version)) biiresponse.info("\tVersion %s (%s) valid" % (version, tag)) else: biiresponse.info("\tVersion %s (%s) discarded" % (version, tag)) return delta_versions def _compute_new(self, block_name, decls, policy, existing_block_names, biiresponse): try: biiresponse.info("Looking for %s..." % block_name) # branches = self._store.read_tracks(block_name) # branches.get_blocks() block_candidates = [block_name + BranchName("%s/master" % block_name.user)] block_candidates = policy.filter(block_candidates) delta_versions = self._filter_by_policy(block_candidates, policy, biiresponse) logger.debug("The heap is %s" % delta_versions) result = self._define_hypothesis(delta_versions, decls, existing_block_names, biiresponse) return result except ForbiddenException: # Propagate forbidden to client raise except NotInStoreException: biiresponse.warn("Can't find block candidate for: %s" % (str(block_name))) return [] except Exception: biiresponse.error("Fatal error in server while reading %s" % block_name) logger.error(traceback.format_exc()) return []
class PublishService(object): ''' Service for publish blocks in server.''' def __init__(self, store, auth_user): self._store = store self.auth_user = auth_user self.security = Security(self.auth_user, self._store) def publish(self, publish_request): '''Performs a publication TIP: If we add publish_request to transaction_definition we can easily have asynchronous publications private: Only for first publication ''' from biicode.server.background.enqueuer import register_publish if publish_request.tag == DEV: if not publish_request: raise BiiRequestErrorException('Up to date, nothing to publish') if publish_request.versiontag is not None: raise PublishException('A DEV version cannot have tag %s' % publish_request.tag) assert publish_request.deptable is not None # by default it is public # TODO: BLock creation is not handled in the transaction target_version = publish_request.parent user = self._store.read_user(target_version.block.owner) # Look if user has the block already created, because the block # can exist with -1 version if it has been created in web if target_version.block not in user.blocks.keys(): try: if target_version != publish_request.parent: # Branching user = self.create_block(target_version.block, publish_request.parent, private=False) else: user = self.create_block(target_version.block, private=False) except DuplicateBlockException: pass # Its ok, already created target_block = target_version.block self._store.requestBlockTransaction(target_block) try: # If we can't read the block, we can't know about his existence self.security.check_read_block(target_block) self.security.check_publish_block(target_block, publish_request) # biiresponse.debug('Read block "%s"' % brl_block) block = self._store.read_block(target_block) (cells, contents, old_cells_ids, old_content_ids) = self._in_memory_block_update(block, publish_request) except ForbiddenException: self._store.finishBlockTransaction(target_block) raise except PublishException as e: self._store.finishBlockTransaction(target_block) raise ServerInternalErrorException(e.message) except Exception as excp: logger.error("Exception in publish service!!: %s " % str(excp)) tb = traceback.format_exc() logger.error(tb) self._store.finishBlockTransaction(target_block) raise ServerInternalErrorException() self._store.beginBlockTransaction(target_block, cells, contents) try: self._write_resources_to_db(cells, contents, old_cells_ids, old_content_ids) self._store.update_block(block) self._store.commitBlockTransaction(target_block) register_publish(self.auth_user, block.last_version()) self._store.finishBlockTransaction(target_block) # Need to read user again, otherwise will raise MongoNotCurrentObjectException # because of double update of same memory object user = self._store.read_user(target_version.block.owner) user.add_block_size_bytes(target_version.block, publish_request.bytes) # Save user (with block bytes updated) self._store.update_user(user) return block.last_version() except Exception as excp: tb = traceback.format_exc() logger.debug(tb) self._rollback_transaction(excp, target_block) raise ServerInternalErrorException('Publish transaction failed. Please, retry') def create_block(self, brl, private=False): '''Creates a block in server due the brl and description''' self.security.check_create_block(brl.owner, private) user = self._store.read_user(brl.owner) try: block_id = user.add_block(brl) # should fail if existing except DuplicateBlockException: logger.debug('Block %s already existing, not creating it' % brl) raise block = Block(block_id, brl) try: # FIXME: better upsert? self._store.create_block(block, private) # should fail if existing except AlreadyInStoreException: pass self._store.update_user(user) # raise exception if not current return user def _rollback_transaction(self, excp, brl_block): '''rollback transaction for publish''' logger.warning(str(excp) + '\nRolling back publish transaction') self._store.rollBackBlockTransaction(brl_block) self._store.finishBlockTransaction(brl_block) def _write_resources_to_db(self, cells, contents, old_cells_ids, old_content_ids): '''Write cells and contents to db''' if old_cells_ids: self._store.delete_published_cells(old_cells_ids) if old_content_ids: self._store.delete_published_contents(old_content_ids) if cells: self._store.create_published_cells(cells) if contents: self._store.create_published_contents(contents) # @mongo_update_if_current_safe_retry # def __update_user_if_current(self, user): def _set_cell_roots(self, block, publish_request): '''Set cell root''' # Ensure here root assignment old_ids = {} deltas = block.deltas last_time = len(deltas) - 2 for res in publish_request.cells: old_name = publish_request.renames.get_old_name(res.name.cell_name) old_id = block.cells.get_id(old_name, last_time) if old_id: old_ids[old_id] = res else: res.root = res.ID old_cells = self._store.read_published_cells(old_ids.keys()) for old_id, old_cell in old_cells.iteritems(): res = old_ids[old_id] res.root = old_cell.root def _in_memory_block_update(self, block, publish_request): '''Updates block in memory''' self.security.check_write_block(block.ID) cells, contents, old_cells_ids, old_content_ids = block.add_publication(publish_request, self.auth_user) self._set_cell_roots(block, publish_request) return cells, contents, old_cells_ids, old_content_ids def get_block_info(self, brl_block): '''Check if auth_user can publish a block version specified by parameter block_version Returns: BlockInfo ''' try: self.security.check_read_block(brl_block) except NotInStoreException: # In this case, the block doesnt exist, but return information of -1 and permissions return self._get_new_block_info(brl_block) block_info = BlockInfo() try: self.security.check_write_block(brl_block) block_info.can_write = True except ForbiddenException: block_info.can_write = False try: block = self._store.read_block(brl_block) block_info.last_version = block.last_version() block_info.private = self.security.is_private(brl_block) except Exception as e: tb = traceback.format_exc() logger.debug(tb) logger.error("Something went wrong with %s" % e) raise BiiServiceException('Something went wrong') return block_info def _get_new_block_info(self, brl_block): ''' Returns BlockInfo that new block would have if we publish it. Raises exception if block cannot be created for any reason ''' last_version = BlockVersion(brl_block, -1) can_write = False try: self.security.check_create_block(brl_block.owner) can_write = True except ForbiddenException: can_write = False except NotInStoreException: raise NotFoundException("Block %s not found!" % brl_block.to_pretty()) return BlockInfo(can_write=can_write, last_version=last_version)
class FindService(object): MAX_HYP = 10 def __init__(self, store, auth_user): self._store = store self._auth_user = auth_user self.security = Security(self._auth_user, self._store) self.translator = ReferenceTranslatorService(self._store, self._auth_user) def find(self, request, biiout): ''' Params: request: FinderRequest biiout: biiout Rerturns: FinderResult ''' if not request: raise ValueError('The find request is empty, nothing to find') logger.debug('---------FinderRequest ------------\n%s' % str(request)) result = FinderResult() # Copy unresolved and remove it if find the dependence result.unresolved = copy(request.unresolved) hypothesis = self._get_hypothesis(request, biiout) if not hypothesis: biiout.info("No block candidates found") return result biiout.info("Analyzing compatibility for found dependencies... ") '''# primitive combinator variant analyzer = CompatibilityAnalyzer(self._store, self._auth_user) analysis_result = analyzer.solve(hypothesis) # standard constraint variant csp = CSPExact(hypothesis, None) csp.solveCSP() analysis_result = csp.getCompatibleSol() logger.info(csp.print_info())''' # iterative deepening variant it = IterDeep(hypothesis, None, None) sol_found, analysis_result = it.start() if sol_found: logger.info("sol found: {0} iter".format(it.num_iter)) if analysis_result is None: biiout.error("Can't find a compatible solution") return result self._update_result(analysis_result, request, result, biiout) if not result.unresolved: if result.resolved: biiout.info('All dependencies resolved') elif not result.updated: biiout.info('Everything was up to date') logger.debug('Result %s' % result) return result def _get_hypothesis(self, request, biiresponse): hypothesis = [] if request.find: # group unresolved declarations by module possible_blocks = request.possible_blocks() logger.debug('Possible blocks %s' % possible_blocks) for block_name, decls in possible_blocks.items(): hyp = self._compute_new(block_name, decls, request.policy, request.block_names, biiresponse) if len(hyp) > 0: # Don't append [] hypothesis.append(hyp) existing_hypothesis = self._compute_existing(request, biiresponse) if len(existing_hypothesis) > 0: hypothesis.extend(existing_hypothesis) logger.debug('Hypothesis %s' % hypothesis) return hypothesis def _update_result(self, analysis_result, request, result, biiout): #existing = {version.block: version.time for version in request.existing} for elem in analysis_result: version = elem.block_version for declaration, refs in elem.dep_dict.iteritems(): if declaration in request.unresolved: biiout.debug("Resolved declaration %s" % str(declaration)) result.resolved[version][declaration] = refs elif version not in request.existing: biiout.info("Block %s updated to version %s" % (version.block, version.time)) result.updated[version][declaration] = refs # Remove cells from finder response unresolved result.unresolved.difference_update(elem.dep_dict.keys()) def _compute_existing(self, request, biiout): '''return a list of list of hypothesis for already defined (existing) dependencies''' result = [] for block_version, deps in request.existing.iteritems(): if request.update or request.downgrade or request.modify: hypothesis = self._compute_modify(block_version, deps, request, biiout) else: hypothesis = [] hypothesis.append( Hypothesis(block_version, request.existing[block_version], self.translator, request.block_names, biiout)) result.append(hypothesis) return result def _compute_modify(self, block_version, dependencies, request, biiout): ''' Params: block_version: Version to which dependencies are currently resolved to dependencies: {Declaration: set(BlockCellName)} request: FinderRequest ''' brl_block = block_version.block time = block_version.time # First, compute all block candidates that have to be considered block_candidates = {brl_block} # Remove those not wanted by our policy policy = request.policy block_candidates = policy.filter(block_candidates) current_block = self._store.read_block(brl_block) original_date = current_block.deltas[time].date delta_versions = self._filter_by_policy(block_candidates, policy, biiout, original_date, request) logger.debug("The heap is %s" % delta_versions) hypothesis = self._define_hypothesis(delta_versions, dependencies, request.block_names, biiout, block_version) return hypothesis def _match_declarations(self, decls, block, snap, cur_version, version): ''' Params: decls: Current declarations for given block block: Block to match snap: dict {CellName => ID} for new version cur_version: Current BlockVersion that decls are resolved to version: New BlockVersion to evaluate Return: all_found: boolean names_total: set(BlockCellName) deps_dict: Dict {Declaration => set(BlockCellName)} ''' all_found = True deps_dict = {} names_total = set() block_name = block.ID.block_name for decl in decls: matchable_names = snap.keys() renames = {} if cur_version: renames = block.get_renames(cur_version.time, version.time) matchable_names.extend( renames.keys()) # Adding old names so decls matches names = decl.match([block_name + name for name in matchable_names]) names = set([ n if n.cell_name in snap.keys() else (block_name + renames[n.cell_name]) for n in names ]) # Apply renames if names: # In case of renames here we will have a mismatch between declaration and cell_name # it will be corrected by client by updating declaration when it detects such # mismatch deps_dict[decl] = names names_total.update(names) else: all_found = False break return all_found, names_total, deps_dict def _define_hypothesis(self, delta_versions, decls, existing_block_names, biiresponse, cur_version=None): ''' Parameters: delta_versions: [(delta, block_version)], prioritized set of accepted hypothesis decls: {Declaration: set(BlockCellName)} existing_block_names = set(BlockName) cur_version: Current version that decls are resolved to Returns: list of hypothesis that match the required decls ''' result = [] #repeated = set() #previous = None for _, version in delta_versions: logger.debug('Analyzing hypothesis %s' % str(version)) block = self._store.read_block(version.block) snap = block.cells.get_all_ids(version.time) #logger.debug('Current snap %s' % snap) all_found, names_total, deps_dict = self._match_declarations( decls, block, snap, cur_version, version) if not all_found: biiresponse.debug( 'Version %s discarded, only contains files for declarations %s' % (str(version), deps_dict.keys())) continue # Store the current IDs and dep table #snap_contents = block.contents.get_ids(version.time) #cell_ids = {snap[k.cell_name] for k in names_total} #content_ids = {snap_contents[k.cell_name] for k in names_total if # k.cell_name in snap_contents} #dep_table = block.dep_tables.floor(version.time) #current = cell_ids, content_ids, dep_table # Only if the current option is different to the previous one # we dont want to check the same option twice #if previous != current and deps_dict: logger.debug('Building hypothesis for %s with %s' % (version, deps_dict)) # logger.debug('ref_dict %s' % ref_dict) hyp = Hypothesis(version, deps_dict, self.translator, existing_block_names, biiresponse) result.append(hyp) #previous = current # FIXME: now the limit of hypothesis is hardwired if len(result) >= FindService.MAX_HYP: break return result def _filter_by_policy(self, block_candidates, policy, biiresponse, original_date=None, request=None): '''computes list of (block_delta, block_version) for each block candidate''' delta_versions = [] for block_candidate in block_candidates: self.security.check_read_block(block_candidate) biiresponse.info("Block candidate: %s" % str(block_candidate)) block = self._store.read_block(block_candidate) # from last to 0, backwards for num_version in range(len(block.deltas) - 1, -1, -1): tag = block.deltas[num_version].tag date = block.deltas[num_version].date if request: if not request.downgrade and date <= original_date: continue version = BlockVersion(block.ID, num_version) ev = policy.evaluate(version, tag) if ev: heappush(delta_versions, ((-date, -num_version), version)) biiresponse.info("\tVersion %s (%s) valid" % (version, tag)) else: biiresponse.info("\tVersion %s (%s) discarded" % (version, tag)) return delta_versions def _compute_new(self, block_name, decls, policy, existing_block_names, biiresponse): try: biiresponse.info("Looking for %s..." % block_name) # branches = self._store.read_tracks(block_name) # branches.get_blocks() block_candidates = [ block_name + BranchName("%s/master" % block_name.user) ] block_candidates = policy.filter(block_candidates) delta_versions = self._filter_by_policy(block_candidates, policy, biiresponse) logger.debug("The heap is %s" % delta_versions) result = self._define_hypothesis(delta_versions, decls, existing_block_names, biiresponse) return result except ForbiddenException: # Propagate forbidden to client raise except NotInStoreException: biiresponse.warn("Can't find block candidate for: %s" % (str(block_name))) return [] except Exception: biiresponse.error("Fatal error in server while reading %s" % block_name) logger.error(traceback.format_exc()) return []
class PublishService(object): ''' Service for publish blocks in server.''' def __init__(self, store, auth_user): self._store = store self.auth_user = auth_user self.security = Security(self.auth_user, self._store) def publish(self, publish_request): '''Performs a publication TIP: If we add publish_request to transaction_definition we can easily have asynchronous publications private: Only for first publication ''' from biicode.server.background.enqueuer import register_publish if publish_request.tag == DEV: if not publish_request: raise BiiRequestErrorException( 'Up to date, nothing to publish') if publish_request.versiontag is not None: raise PublishException('A DEV version cannot have tag %s' % publish_request.tag) assert publish_request.deptable is not None # by default it is public # TODO: BLock creation is not handled in the transaction target_version = publish_request.parent user = self._store.read_user(target_version.block.owner) # Look if user has the block already created, because the block # can exist with -1 version if it has been created in web if target_version.block not in user.blocks.keys(): try: if target_version != publish_request.parent: # Branching user = self.create_block(target_version.block, publish_request.parent, private=False) else: user = self.create_block(target_version.block, private=False) except DuplicateBlockException: pass # Its ok, already created target_block = target_version.block self._store.requestBlockTransaction(target_block) try: # If we can't read the block, we can't know about his existence self.security.check_read_block(target_block) self.security.check_publish_block(target_block, publish_request) # biiresponse.debug('Read block "%s"' % brl_block) block = self._store.read_block(target_block) (cells, contents, old_cells_ids, old_content_ids) = self._in_memory_block_update( block, publish_request) except ForbiddenException: self._store.finishBlockTransaction(target_block) raise except PublishException as e: self._store.finishBlockTransaction(target_block) raise ServerInternalErrorException(e.message) except Exception as excp: logger.error("Exception in publish service!!: %s " % str(excp)) tb = traceback.format_exc() logger.error(tb) self._store.finishBlockTransaction(target_block) raise ServerInternalErrorException() self._store.beginBlockTransaction(target_block, cells, contents) try: self._write_resources_to_db(cells, contents, old_cells_ids, old_content_ids) self._store.update_block(block) self._store.commitBlockTransaction(target_block) register_publish(self.auth_user, block.last_version()) self._store.finishBlockTransaction(target_block) # Need to read user again, otherwise will raise MongoNotCurrentObjectException # because of double update of same memory object user = self._store.read_user(target_version.block.owner) user.add_block_size_bytes(target_version.block, publish_request.bytes) # Save user (with block bytes updated) self._store.update_user(user) return block.last_version() except Exception as excp: tb = traceback.format_exc() logger.debug(tb) self._rollback_transaction(excp, target_block) raise ServerInternalErrorException( 'Publish transaction failed. Please, retry') def create_block(self, brl, private=False): '''Creates a block in server due the brl and description''' self.security.check_create_block(brl.owner, private) user = self._store.read_user(brl.owner) try: block_id = user.add_block(brl) # should fail if existing except DuplicateBlockException: logger.debug('Block %s already existing, not creating it' % brl) raise block = Block(block_id, brl) try: # FIXME: better upsert? self._store.create_block(block, private) # should fail if existing except AlreadyInStoreException: pass self._store.update_user(user) # raise exception if not current return user def _rollback_transaction(self, excp, brl_block): '''rollback transaction for publish''' logger.warning(str(excp) + '\nRolling back publish transaction') self._store.rollBackBlockTransaction(brl_block) self._store.finishBlockTransaction(brl_block) def _write_resources_to_db(self, cells, contents, old_cells_ids, old_content_ids): '''Write cells and contents to db''' if old_cells_ids: self._store.delete_published_cells(old_cells_ids) if old_content_ids: self._store.delete_published_contents(old_content_ids) if cells: self._store.create_published_cells(cells) if contents: self._store.create_published_contents(contents) # @mongo_update_if_current_safe_retry # def __update_user_if_current(self, user): def _set_cell_roots(self, block, publish_request): '''Set cell root''' # Ensure here root assignment old_ids = {} deltas = block.deltas last_time = len(deltas) - 2 for res in publish_request.cells: old_name = publish_request.renames.get_old_name(res.name.cell_name) old_id = block.cells.get_id(old_name, last_time) if old_id: old_ids[old_id] = res else: res.root = res.ID old_cells = self._store.read_published_cells(old_ids.keys()) for old_id, old_cell in old_cells.iteritems(): res = old_ids[old_id] res.root = old_cell.root def _in_memory_block_update(self, block, publish_request): '''Updates block in memory''' self.security.check_write_block(block.ID) cells, contents, old_cells_ids, old_content_ids = block.add_publication( publish_request, self.auth_user) self._set_cell_roots(block, publish_request) return cells, contents, old_cells_ids, old_content_ids def get_block_info(self, brl_block): '''Check if auth_user can publish a block version specified by parameter block_version Returns: BlockInfo ''' try: self.security.check_read_block(brl_block) except NotInStoreException: # In this case, the block doesnt exist, but return information of -1 and permissions return self._get_new_block_info(brl_block) block_info = BlockInfo() try: self.security.check_write_block(brl_block) block_info.can_write = True except ForbiddenException: block_info.can_write = False try: block = self._store.read_block(brl_block) block_info.last_version = block.last_version() block_info.private = self.security.is_private(brl_block) except Exception as e: tb = traceback.format_exc() logger.debug(tb) logger.error("Something went wrong with %s" % e) raise BiiServiceException('Something went wrong') return block_info def _get_new_block_info(self, brl_block): ''' Returns BlockInfo that new block would have if we publish it. Raises exception if block cannot be created for any reason ''' last_version = BlockVersion(brl_block, -1) can_write = False try: self.security.check_create_block(brl_block.owner) can_write = True except ForbiddenException: can_write = False except NotInStoreException: raise NotFoundException("Block %s not found!" % brl_block.to_pretty()) return BlockInfo(can_write=can_write, last_version=last_version)
def test_anonymous_operations(self): ensure = Security(None, self.store) ensure.check_read_block(self.public_brl)
def test_read_block(self): # 1. Check we can always access a public block brl = BRLBlock("goku/goku/block/master") bper = ElementPermissions(brl, private=False) self.store.upsert_block_permissions(bper) ensure = Security("freezer", self.store) self._subscribe("freezer", "enterprise_275_50_x") ensure.check_read_block(brl) # Even owner is not paying self._subscribe("freezer", "free") ensure.check_read_block(brl) self._subscribe("freezer", "enterprise_275_50_x") # 2. Check we can read a private block due to being the owner bper = ElementPermissions(brl, private=True) self.store.upsert_block_permissions(bper) ensure = Security("goku", self.store) self._subscribe("freezer", "enterprise_275_50_x") ensure.check_read_block(brl) # 4. Check we can read the block even subscription is not valid self._subscribe("freezer", "free") ensure.check_read_block(brl) # 5. A user without read permissions cant read the block ensure = Security("bulma", self.store) self._subscribe("bulma", "enterprise_275_50_x") self.assertRaises(ForbiddenException, ensure.check_read_block, brl) # 6. Until is granted as administrator or read granted goku = self.store.read_user("goku") goku.administrators.grant("bulma") self.store.update_user(goku) ensure.check_read_block(brl) goku.administrators.revoke("bulma") self.store.update_user(goku) self.assertRaises(ForbiddenException, ensure.check_read_block, brl) bper = ElementPermissions(brl, private=True) bper.read.grant("bulma") self.store.upsert_block_permissions(bper) ensure.check_read_block(brl) bper.read.remove("bulma") self.store.upsert_block_permissions(bper) self.assertRaises(ForbiddenException, ensure.check_read_block, brl)
def test_publish(self, enqueuer): brl = BRLBlock('owner/user/block/branch') # moduleID=BlockID(UserID(123),456) store = Mock(MongoServerStore) store.read_block_permissions = Mock( return_value=ElementPermissions(brl, private=False)) user = User("owner") user.numeric_id = 1 user.blocks = {} store.read_user = Mock(return_value=user) block = Mock(Block) block.last_version.return_value = Mock(BlockVersion) block.add_publication.return_value = (Mock(list), Mock(list), Mock(list), Mock(list)) block.deltas = [] ensure = Security('authUser', store) ensure.check_create_block = Mock(return_value=True) ensure.check_write_block = Mock(return_value=True) ensure.check_read_block = Mock(return_value=True) store.read_block.return_value = block store.read_published_cells.return_value = {} p = PublishService(store, 'authUser') p.security = ensure pack = PublishRequest(BlockVersion(brl, -1)) pack.cells.append(SimpleCell('user/block/r1.h')) pack.contents['r1.h'] = Content(id_=None, load=Blob('hola')) pack.cells.append(SimpleCell('user/block/r2.h')) pack.contents['r2.h'] = Content(id_=None, load=Blob('hola')) pack.cells.append(SimpleCell('user/block/r3.h')) pack.contents['r3.h'] = Content(id_=None, load=Blob('hola')) pack.deptable = BlockVersionTable() p.publish(pack) block.add_publication.assert_called_once_with(pack, p.auth_user) store.update_block.assert_called_once_with(block) self.assertEqual(1, store.create_published_cells.call_count) self.assertEqual(1, store.create_published_contents.call_count) # Check sizes self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3 # Publish again, see the size incremented pack._bytes = None # Lazy computed p.publish(pack) self.assertEquals(user.blocks_bytes, 24) # 24 bytes: "hola" * 3 * 2 publications # Max size exceeded for user user.max_workspace_size = 25 self.assertRaises(ForbiddenException, p.publish, pack) # Try to publish only 1 byte pack._bytes = None # Lazy computed pack.cells = [] pack.contents = {} pack.cells.append(SimpleCell('user/block/r1.h')) pack.contents['r1.h'] = Content(id_=None, load=Blob('h')) p.publish(pack)
class ReferenceTranslatorService(object): def __init__(self, store, auth_user): self._store = store self.security = Security(auth_user, self._store) def get_published_resources(self, references): result = ReferencedResources() for block_version, cell_names in references.iteritems(): try: self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) cell_ids = block.cells.get_ids(cell_names, block_version.time) content_ids = block.contents.get_ids(cell_names, block_version.time) cells = self._store.read_published_cells(cell_ids.values()) contents = self._store.read_published_contents( content_ids.values()) for name, rID in cell_ids.iteritems(): if name in content_ids: cid = content_ids[name] cid = contents[cid] else: cid = None # Virtual resource result[block_version][name] = Resource(cells[rID], cid) except (ForbiddenException, NotInStoreException): pass return result def get_published_min_refs(self, references): '''returns the minimum information required to perform a compatibility check for those references. This method is currently used just by CompatibilityClosureBuilder param references: {block_version: set(cell_names)} return: {block_version: {cell_name: (cell_id, content_id), root_id, [deps blockcellnames]}} ''' result = defaultdict(dict) for block_version, cell_names in references.iteritems(): try: self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) cell_ids = block.cells.get_ids(cell_names, block_version.time) content_ids = block.contents.get_ids(cell_names, block_version.time) cells = self._store.read_min_cells(cell_ids.values()) #This cells are {cellID: (rootID, dep_block_names)} for cell_name, cell_id in cell_ids.iteritems(): content_id = content_ids.get( cell_name) # None if Virtual resource root_id, deps = cells.get(cell_id, (None, None)) if root_id is not None: result[block_version][cell_name] = ((cell_id, content_id), root_id, deps) except (ForbiddenException, NotInStoreException): pass return result def get_dep_table(self, block_version): self.security.check_read_block(block_version.block) block = self._store.read_block(block_version.block) table = block.dep_tables.find(block_version.time) return table