def virtual_test(self): references = References() references[va].add('a.h') code = ( 'def virtual(settings):\n\tif(settings.os.family == "windows"):return "win"\n' '\telse: return "nix"') a = Resource(VirtualCell('user/blocka/a.h', code, {'win', 'nix'})) awin = Resource(SimpleCell('user/blocka/win/a.h')) anix = Resource(SimpleCell('user/blocka/nix/a.h')) b = Resource(SimpleCell('user/blockb/b.h')) d1 = Resource(SimpleCell('user/blockd/d.h')) awin.cell.dependencies.explicit.add(b.name) anix.cell.dependencies.explicit.add(d1.name) tables = {va: [vb, vd1], vb: [], vd1: []} api = FakeApi(zip([va, va, va, vb, vd1], [a, awin, anix, b, d1]), tables) #With windows settings settings = Settings(OSInfo(OSFamily("Windows"))) biiout = OutputStream() graph, closure, _ = build_closure(api, references, {}, settings, biiout) expected_graph = BlockVersionGraph() expected_graph.add_nodes([va, vb]) expected_graph.add_edge(va, vb) self.assertEqual(expected_graph, graph) expected_closure = Closure({ a.name: ClosureItem(a, va), awin.name: ClosureItem(awin, va), b.name: ClosureItem(b, vb), }) self.assertEqual(expected_closure, closure) self.assertEqual("", str(biiout)) #Change settings settings = Settings(OSInfo(OSFamily("Linux"))) biiout = OutputStream() graph, closure, _ = build_closure(api, references, {}, settings, biiout) expected_graph = BlockVersionGraph() expected_graph.add_nodes([va, vd1]) expected_graph.add_edge(va, vd1) self.assertEqual(expected_graph, graph) expected_closure = Closure({ a.name: ClosureItem(a, va), anix.name: ClosureItem(anix, va), d1.name: ClosureItem(d1, vd1), }) self.assertEqual(expected_closure, closure) self.assertEqual("", str(biiout))
def __init__(self): self.dep_table = None # BlockVersionTable() self.references = None # References() # TODO: The whole closure is being serialized. Check performance, improve self.closure = Closure() # {BlockCellName: Resource} self.src_graph = BlockVersionGraph() # BlockVersionGraph self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def overwrite_test(self): """ Blocks: C (defines in requirements B2) Deps: A, B2 C -> A -> B1 """ references = References() references[va].add('a.h') b = Resource(SimpleCell('user/blockb/b.h')) b2 = Resource(SimpleCell('user/blockb/b.h'), CPP) a = Resource(SimpleCell('user/blocka/a.h')) a2 = Resource(SimpleCell('user/blocka/a2.h')) a.cell.dependencies.explicit.add(a2.name) a2.cell.dependencies.explicit.add(b.name) tables = {va: [vb], vb: [], vb2: []} api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables) base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here biiout = OutputStream() graph, closure, _ = build_closure(api, references, base_table, biiout=biiout) expected_graph = BlockVersionGraph() expected_graph.add_nodes([va, vb2]) expected_graph.add_edge(va, vb2) self.assertEqual(expected_graph, graph) expected_closure = Closure({ a.name: ClosureItem(a, va), a2.name: ClosureItem(a2, va), b.name: ClosureItem(b2, vb2) }) self.assertEqual(expected_closure, closure) self.assertEqual("", str(biiout))
def simple_test(self): """ Blocks: C Deps: A, B C -> A -> B """ references = References() references[va].add('a.h') base_table = BlockVersionTable( [vc, va]) # The result including or excluding va is same b = Resource(SimpleCell('user/blockb/b.h')) a = Resource(SimpleCell('user/blocka/a.h')) a2 = Resource(SimpleCell('user/blocka/a2.h')) a.cell.dependencies.explicit.add(a2.name) a2.cell.dependencies.explicit.add(b.name) tables = {va: [vb], vb: []} api = FakeApi(zip([va, va, vb], [a, a2, b]), tables) biiout = OutputStream() graph, closure, _ = build_closure(api, references, base_table, biiout=biiout) expected_graph = BlockVersionGraph() expected_graph.add_nodes([va, vb]) expected_graph.add_edge(va, vb) self.assertEqual(expected_graph, graph) expected_closure = Closure({ a.name: ClosureItem(a, va), a2.name: ClosureItem(a2, va), b.name: ClosureItem(b, vb) }) self.assertEqual(expected_closure, closure) self.assertEqual("", str(biiout))
def dep_overwriten_in_blocks_test(self): """ Blocks: C, B (None version) Deps: A C -> A -> B """ references = References() references[va].add('a.h') vbn = BlockVersion(vb.block, None) b = Resource(SimpleCell('user/blockb/b.h')) a = Resource(SimpleCell('user/blocka/a.h')) a2 = Resource(SimpleCell('user/blocka/a2.h')) a.cell.dependencies.explicit.add(a2.name) a2.cell.dependencies.explicit.add(b.name) tables = {va: [vb], vb: []} api = FakeApi(zip([va, va, vb], [a, a2, b]), tables) base_table = BlockVersionTable([vc, vbn]) biiout = OutputStream() graph, closure, _ = build_closure(api, references, base_table, biiout=biiout) expected_graph = BlockVersionGraph() expected_graph.add_nodes([va]) expected_graph.add_edge(va, vbn) self.assertEqual(expected_graph, graph) expected_closure = Closure({ a.name: ClosureItem(a, va), a2.name: ClosureItem(a2, va) }) self.assertEqual(expected_closure, closure) self.assertEqual("", str(biiout))
def deserialize(data): try: res = HiveDependencies() res.dep_table = BlockVersionTable.deserialize(data[HiveDependencies.SERIAL_DEP_TABLE]) res.references = References.deserialize(data[HiveDependencies.SERIAL_REFERENCES]) res.closure = Closure.deserialize(data[HiveDependencies.SERIAL_CLOSURE]) res.src_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_SRC_GRAPH]) res.dep_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_DEP_GRAPH]) return res except Exception as e: raise BiiSerializationException(e)
def deserialize(data): try: res = HiveDependencies() res.dep_table = BlockVersionTable.deserialize( data[HiveDependencies.SERIAL_DEP_TABLE]) res.references = References.deserialize( data[HiveDependencies.SERIAL_REFERENCES]) res.closure = Closure.deserialize( data[HiveDependencies.SERIAL_CLOSURE]) res.src_graph = BlockVersionGraph.deserialize( data[HiveDependencies.SERIAL_SRC_GRAPH]) res.dep_graph = BlockVersionGraph.deserialize( data[HiveDependencies.SERIAL_DEP_GRAPH]) return res except Exception as e: raise BiiSerializationException(e)
def test_add_item(self): c1 = Closure() r1 = Cell('user/block/name1') version = BlockVersion(BRLBlock('owner/user/block/branch'), 13) biiout = OutputStream() resource = Resource(r1, None) c1.add_item(resource, version, biiout) r2 = Cell('user/block/name1') r2.hasMain = True resource2 = Resource(r2, None) version2 = BlockVersion(BRLBlock('owner/user/block/branch'), 14) c1.add_item(resource2, version2, biiout) self.assertEqual(ClosureItem(resource, version), c1['user/block/name1']) self.assertIn('Incompatible dependency', str(biiout))
class HiveDependencies(object): """ This is a caching object for storing the block and version level dependencies of the current graph, as well as the closure """ def __init__(self): self.references = None # References() self.closure = Closure() # {BlockCellName: Resource} self.src_graph = BlockVersionGraph() # BlockVersionGraph self.dep_graph = BlockVersionGraph() # BlockVersionGraph @property def version_graph(self): return self.src_graph + self.dep_graph @property def bii_config(self): ''' Return a dict of biicode.conf objects, e.g.: {user/superblock: <bii_config_object>, user_2/otherblock: <bii_config_object>} ''' return self.closure.bii_config()
def build_closure(biiapi, references, base_table, settings=None, biiout=None): graph = BlockVersionGraph() closure = Closure() visited = set() visited_block_versions = set() overwrites = defaultdict(set) if references: assert not ([ r for r in references.explode() if r.block_version.block_name in base_table and base_table[r.block_version.block_name] != r.block_version ]) frontier = [(references, base_table) ] # Of tuple (references, base_table) else: frontier = [] class Visited(namedtuple('Visited', 'version cell_name, dep_table')): def __hash__(self): return hash((self.version, self.cell_name)) while frontier: references, base_table = frontier.pop() retrieved = biiapi.get_published_resources(references) for block_version, resources in retrieved.iteritems(): graph.add_node(block_version) dep_targets = set() if block_version not in visited_block_versions: cell_names = biiapi.get_cells_snapshot(block_version) _add_implicit_targets(dep_targets, cell_names, block_version) visited_block_versions.add(block_version) try: up_table = biiapi.get_dep_table(block_version) except Exception as e: raise BiiException( '%s\nbiicode needs compare your local "%s" block with your last' ' version published one. If you want to delete it, delete the ' 'folder in the filesystem.' % (str(e), block_version.block_name)) effective, propagate, overwrite = compute_effective( base_table, up_table, block_version.block_name) if overwrite: overwrites[block_version].update(overwrite) for cell_name, resource in resources.iteritems(): closure.add_item(resource, block_version, biiout) visited.add(Visited(block_version, cell_name, base_table)) _update_dep_targets(dep_targets, resource, settings, biiout) other_refs = References() self_refs = References() for target in dep_targets: if target.block_name == block_version.block_name: next_visit = Visited(block_version, target.cell_name, base_table) if next_visit not in visited: self_refs[block_version].add(target.cell_name) else: next_version = effective[target.block_name] graph.add_edge(block_version, next_version) if next_version.time is not None: next_visit = Visited(next_version, target.cell_name, propagate) if next_visit not in visited: other_refs[next_version].add(target.cell_name) if other_refs: frontier.append((other_refs, propagate)) if self_refs: frontier.append((self_refs, base_table)) return graph, closure, overwrites
def __init__(self): self.references = None # References() self.closure = Closure() # {BlockCellName: Resource} self.src_graph = BlockVersionGraph() # BlockVersionGraph self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def build_closure(biiapi, references, base_table, settings=None, biiout=None): graph = BlockVersionGraph() closure = Closure() visited = set() visited_block_versions = set() overwrites = defaultdict(set) if references: assert not ([r for r in references.explode() if r.block_version.block_name in base_table and base_table[r.block_version.block_name] != r.block_version]) frontier = [(references, base_table)] # Of tuple (references, base_table) else: frontier = [] class Visited(namedtuple('Visited', 'version cell_name, dep_table')): def __hash__(self): return hash((self.version, self.cell_name)) while frontier: references, base_table = frontier.pop() retrieved = biiapi.get_published_resources(references) for block_version, resources in retrieved.iteritems(): graph.add_node(block_version) dep_targets = set() if block_version not in visited_block_versions: cell_names = biiapi.get_cells_snapshot(block_version) _add_implicit_targets(dep_targets, cell_names, block_version) visited_block_versions.add(block_version) try: up_table = biiapi.get_dep_table(block_version) except Exception as e: raise BiiException('%s\nbiicode needs compare your local "%s" block with your last' ' version published one. If you want to delete it, delete the ' 'folder in the filesystem.' % (str(e), block_version.block_name)) effective, propagate, overwrite = compute_effective(base_table, up_table, block_version.block_name) if overwrite: overwrites[block_version].update(overwrite) for cell_name, resource in resources.iteritems(): closure.add_item(resource, block_version, biiout) visited.add(Visited(block_version, cell_name, base_table)) _update_dep_targets(dep_targets, resource, settings, biiout) other_refs = References() self_refs = References() for target in dep_targets: if target.block_name == block_version.block_name: next_visit = Visited(block_version, target.cell_name, base_table) if next_visit not in visited: self_refs[block_version].add(target.cell_name) else: next_version = effective[target.block_name] graph.add_edge(block_version, next_version) if next_version.time is not None: next_visit = Visited(next_version, target.cell_name, propagate) if next_visit not in visited: other_refs[next_version].add(target.cell_name) if other_refs: frontier.append((other_refs, propagate)) if self_refs: frontier.append((self_refs, base_table)) return graph, closure, overwrites