def requirements_update_test(self):
block_name = BlockName("user/block")
text = "[requirements]\n # My comment\n"
config = BiiConfig(text)
self.assertEqual(config.dumps(), None)
version = BlockVersion(block_name.default_block(), 0)
config.requirements = BlockVersionTable([version])
dumped = config.dumps()
self.assertEqual(dumped, "[requirements]\n\t user/block: 0\n\n")
# Dump again, no changes
self.assertEqual(config.dumps(), None)
config2 = BiiConfig(dumped)
self.assertEqual(config2.requirements, config.requirements)
block_name2 = BlockName("auser2/block2")
version2 = BlockVersion(block_name2.default_block(), 2)
config.requirements = BlockVersionTable([version, version2])
dumped = config.dumps()
self.assertEqual(
dumped, "[requirements]\n"
"\t auser2/block2: 2\n\t user/block: 0\n\n")
# Dump again, no changes
self.assertEqual(config.dumps(), None)
config3 = BiiConfig(dumped)
self.assertEqual(config3.requirements, config.requirements)
def versions(self):
""" given a set of block_names in edition (blocks folder), and a tracking BlockVersionTable
return the current versions of such edition blocks, that is, time = None
return: BlockVersionTable{ BlockName: BlockVersion time=None}
"""
edition_versions = BlockVersionTable()
for block_holder in self._block_holders.itervalues():
parent = block_holder.parent
edition_versions.add_version(BlockVersion(parent.block, None))
return edition_versions
def versions(self):
""" given a set of block_names in edition (blocks folder), and a tracking BlockVersionTable
return the current versions of such edition blocks, that is, time = None
return: BlockVersionTable{ BlockName: BlockVersion time=None}
"""
edition_versions = BlockVersionTable()
for block_holder in self._block_holders.itervalues():
parent = block_holder.parent
edition_versions.add_version(BlockVersion(parent.block, None))
return edition_versions
def test_basic(self):
brl1 = BRLBlock("user/user2/block/branch")
v1 = BlockVersion(brl1, 1)
brl2 = BRLBlock("user2/user3/block2/branch3")
v2 = BlockVersion(brl2, 2)
table = BlockVersionTable()
table.add_version(v1)
table.add_version(v2)
self.assertEqual(v1, table['user2/block'])
self.assertEqual(v2, table['user3/block2'])
self.assertEqual(2, len(table))
def test_basic(self):
brl1 = BRLBlock("user/user2/block/branch")
v1 = BlockVersion(brl1, 1)
brl2 = BRLBlock("user2/user3/block2/branch3")
v2 = BlockVersion(brl2, 2)
table = BlockVersionTable()
table.add_version(v1)
table.add_version(v2)
self.assertEqual(v1, table['user2/block'])
self.assertEqual(v2, table['user3/block2'])
self.assertEqual(2, len(table))
def _compute_frontier(missing_dict, restricted_versions, full_graph, retrieved, open_frontier):
frontier = References()
for block_version, targets in missing_dict.iteritems():
neighbours = BlockVersionTable(full_graph.neighbours(block_version))
for target in targets:
if target.block_name != block_version.block_name:
other_version = neighbours.get(target.block_name)
if other_version not in restricted_versions:
open_frontier[other_version].add(target.cell_name)
else:
other_version = block_version
if other_version in restricted_versions and target.cell_name not in retrieved[other_version]:
frontier[other_version].add(target.cell_name)
return frontier
def base_version_test(self):
hive_holder = HiveHolder({}, {})
parents_resource = Resource(
SimpleCell(a1.block_name + BIICODE_FILE),
Content(id_=None, load=Blob('[parent]\n ' + str(a1))))
hive_holder.add_holder(BlockHolder(a1.block_name, {parents_resource}))
parents_resource = Resource(
SimpleCell(b2.block_name + BIICODE_FILE),
Content(id_=None, load=Blob('[parent]\n * ' + str(b2))))
hive_holder.add_holder(BlockHolder(b2.block_name, {parents_resource}))
hive_holder.add_holder(BlockHolder(cn.block_name, {}))
result_table = BlockVersionTable(
[b.parent for b in hive_holder.block_holders])
self.assertEqual(result_table, BlockVersionTable([a1, b2, cn]))
def empty_test(self):
config = BiiConfig("")
self.assertEqual(config.parent, None)
self.assertEqual(config.requirements, BlockVersionTable())
self.assertEqual(config.mains, [])
self.assertEqual(config.dependencies, [])
self.assertEqual(None, config.dumps())
def overwrite_test(self):
""" Blocks: C (defines in requirements B2)
Deps: A, B2
C -> A -> B1
"""
references = References()
references[va].add('a.h')
b = Resource(SimpleCell('user/blockb/b.h'))
b2 = Resource(SimpleCell('user/blockb/b.h'), CPP)
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: [], vb2: []}
api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables)
base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb2])
expected_graph.add_edge(va, vb2)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b2, vb2)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_publish_dev_with_tag(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(
return_value=ElementPermissions(brl, private=False))
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (['mock_id'], [], [], [])
block.deltas = []
ensure = Security('authUser', store)
ensure.check_read_block = Mock(return_value=True)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_publish_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.deptable = BlockVersionTable()
p.publish(pack)
def _compute_frontier(missing_dict, restricted_versions, full_graph, retrieved,
open_frontier):
frontier = References()
for block_version, targets in missing_dict.iteritems():
neighbours = BlockVersionTable(full_graph.neighbours(block_version))
for target in targets:
if target.block_name != block_version.block_name:
other_version = neighbours.get(target.block_name)
if other_version not in restricted_versions:
open_frontier[other_version].add(target.cell_name)
else:
other_version = block_version
if other_version in restricted_versions and target.cell_name not in retrieved[
other_version]:
frontier[other_version].add(target.cell_name)
return frontier
def setUp(self):
self.folder = self.new_tmp_folder()
brl_block = BRLBlock('dummy/dummy/block/master')
self.block_version = BlockVersion(brl_block, 0)
alf = Resource(SimpleCell("dummy/block/alf.c"),
Content(ID((0, 1, 2)), Blob("Hello Alf")))
alf.cell.ID = ID((0, 1, 2))
willy = Resource(SimpleCell("dummy/block/willy.c"),
Content(ID((0, 1, 3)), Blob("Hello Willy")))
willy.cell.ID = ID((0, 1, 45))
self.referenced_resources = ReferencedResources()
self.referenced_resources[self.block_version].update({
CellName("alf.c"):
alf,
CellName("willy.c"):
willy
})
self.cells_snapshot = [CellName("alf.c"), CellName("willy.c")]
self.dep_table = BlockVersionTable()
self.restapi = Mock(BiiAPI)
self.restapi.get_published_resources.return_value = self.referenced_resources
self.restapi.get_cells_snapshot.return_value = self.cells_snapshot
self.restapi.get_dep_table.return_value = self.dep_table
self.restapi.get_version_delta_info.return_value = BlockDelta(
'', DEV, None)
self.localdb = LocalDB(os.path.join(self.folder, 'bii.db'))
self.proxy = BiiAPIProxy(self.localdb, self.restapi, Mock())
def deserialize(data):
'''From dictionary to object Publish Pack'''
pp = PublishRequest()
pp.parent = BlockVersion.deserialize(
data[PublishRequest.SERIAL_TRACKED_KEY])
pp.parent_time = data[PublishRequest.SERIAL_PARENT_DATETIME]
pp.tag = VersionTag.deserialize(data[PublishRequest.SERIAL_TAG_KEY])
pp.msg = data[PublishRequest.SERIAL_MSG_KEY]
# Backward client compatibility
pp.versiontag = data.get(PublishRequest.SERIAL_VTAG_KEY, None)
pp.deptable = BlockVersionTable.deserialize(
data[PublishRequest.SERIAL_DEP_TABLE])
pp.cells = ListDeserializer(CellDeserializer(BlockCellName)).\
deserialize(data[PublishRequest.SERIAL_CELLS_KEY])
pp.deleted = ListDeserializer(CellName).\
deserialize(data[PublishRequest.SERIAL_DELETED_KEY])
pp.renames = Renames.deserialize(
data[PublishRequest.SERIAL_RENAMES_KEY])
pp.contents = DictDeserializer(CellName, ContentDeserializer(BlockCellName)).\
deserialize(data[PublishRequest.SERIAL_CONTENTS_KEY])
pp.contents_ids = DictDeserializer(CellName, ID).\
deserialize(data[PublishRequest.SERIAL_CONTENTS_ID_KEY])
# Backward client compatibility
pp.origin = OriginInfo.deserialize(
data.get(PublishRequest.SERIAL_ORIGIN_INFO, None))
return pp
def simple_test(self):
""" Blocks: C
Deps: A, B
C -> A -> B
"""
references = References()
references[va].add('a.h')
base_table = BlockVersionTable(
[vc, va]) # The result including or excluding va is same
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b, vb)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_loads_dumps(self):
#testing loads
table = BlockVersionTable.loads(test_dep_table1)
self.assertEqual(expected_dep_table1, table)
#dumps without file reference
dump = table.dumps()
self.assertIn('User1/geom: 3', dump)
self.assertIn('user2/ge-om2: 4', dump)
self.assertIn('user3/geom3: 5', dump)
self.assertIn('user4/geom4(user5/branch): 6', dump)
#dump with file, without changes
text = table.dumps(test_dep_table1)
self.assertEqual(test_dep_table1_normalized, text)
#dump with changes
table['User1/geom'] = BlockVersion('user1/user1/geom/branch2', 13)
#dump with file, without changes
expected = '\n'.join([
'# My comment', 'user1/geom(branch2): 13', 'user2/ge-om2: 4', '',
'# My comment2', 'user3/geom3: 5', 'user4/geom4(user5/branch): 6',
''
])
text = table.dumps(text)
self.assertEqual(expected, text)
def _publishDependentCell(self, block_id):
low, high = self._biigraph._get_range_block(block_id)
count = 0
d = self.lbcn
bcn = d[block_id]
brl = self._getBlockBRL(bcn)
for row in range(low, high + 1):
r = SimpleCell(d[block_id])
r.root = ID((0, bcn.block_name[-1:], 0))
deps = DependencySet()
dep_table = BlockVersionTable()
for block in range(block_id):
time = self._biigraph._get_dep_elem_offset(row, block)
if time != -1:
deps.add_implicit(d[block])
dep_table[d[block].block_name] = \
BlockVersion(self._getBlockBRL(d[block]), time)
r.dependencies = deps
pack = PublishRequest(BlockVersion(brl, count - 1))
pack.tag = STABLE
count += 1
pack.cells.append(r)
pack.contents[r.name.cell_name] = Content(
id_=None, load=Blob('hola {0}'.format(r.name.cell_name)))
pack.deptable = dep_table
self.service.publish(pack)
def test_loads_dumps(self):
#testing loads
table = BlockVersionTable.loads(test_dep_table1)
self.assertEqual(expected_dep_table1, table)
#dumps without file reference
dump = table.dumps()
self.assertIn('User1/geom: 3', dump)
self.assertIn('user2/ge-om2: 4', dump)
self.assertIn('user3/geom3: 5', dump)
self.assertIn('user4/geom4(user5/branch): 6', dump)
#dump with file, without changes
text = table.dumps(test_dep_table1)
self.assertEqual(test_dep_table1_normalized, text)
#dump with changes
table['User1/geom'] = BlockVersion('user1/user1/geom/branch2', 13)
#dump with file, without changes
expected = '\n'.join(['# My comment',
'user1/geom(branch2): 13',
'user2/ge-om2: 4',
'',
'# My comment2',
'user3/geom3: 5',
'user4/geom4(user5/branch): 6',
''
])
text = table.dumps(text)
self.assertEqual(expected, text)
def test_store_dep_table(self):
original_deptable = BlockVersionTable()
brl_block = BRLBlock('dummy/dummy/block/master')
block_version = BlockVersion(brl_block, 0)
self.db.set_dep_table(block_version, original_deptable)
retrieved_snap = self.db.get_dep_table(block_version)
self.assertEquals(original_deptable, retrieved_snap)
def dep_overwriten_in_blocks_test(self):
""" Blocks: C, B (None version)
Deps: A
C -> A -> B
"""
references = References()
references[va].add('a.h')
vbn = BlockVersion(vb.block, None)
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
base_table = BlockVersionTable([vc, vbn])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va])
expected_graph.add_edge(va, vbn)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_block_version_table(self):
mv = BlockVersionTable()
mv.add_version(BlockVersion('user/user/block/master', 1))
mv.add_version(BlockVersion('user/user/block/master', 1))
mv.add_version(BlockVersion('user/user/block/master', 1))
s = mv.serialize()
mv2 = BlockVersionTable.deserialize(s)
self.assertEqual(mv, mv2)
def _update_requirements(hive_holder, real_graph, overwrites, common_table,
response):
""" After all deps processing, the requirements of each edited block needs to be updated.
Unused versions can be removed, but also versions might be added, as they are actually
necessary to define the current dependency graph of a given edited block
"""
versions = hive_holder.versions
for block_holder in hive_holder.block_holders:
# First compute a new version of the requirements, which is a partial version of common
new_table = common_table.copy()
current_version = versions[block_holder.block_name]
neighbours = real_graph.neighbours(current_version)
neighbours_table = BlockVersionTable(neighbours)
assert len(neighbours) == len(
neighbours_table) # Neighbors cant be different versions
closure = real_graph.compute_closure(current_version)
closure_versions = defaultdict(set)
for version in closure:
closure_versions[version.block_name].add(version)
for block_name, version in new_table.items():
if block_name not in closure_versions:
new_table.pop(block_name)
elif block_name not in neighbours_table:
delete = True
for over_base, over_versions in overwrites.iteritems():
if over_base in closure and version in over_versions:
delete = False
break
if delete:
new_table.pop(block_name)
else:
assert version == neighbours_table[block_name]
pass
# Update the table, and inform the user of any automatic changes done
for block_name, version in new_table.iteritems():
if version.time is None:
version = hive_holder[block_name].parent
assert version.time is not None
new_table.add_version(version)
dep_table = block_holder.requirements
if new_table != dep_table:
# Inform about added references
for block_name, version in new_table.iteritems():
if block_name not in dep_table:
response.warn('Adding to %s "requirements" version %s' %
(block_holder.block_name, str(version)))
#inform about deleted references
for block_name, version in dep_table.iteritems():
if block_name not in new_table:
response.warn(
'Removing unused reference to "%s" from %s "requirements"'
% (str(version), block_holder.block_name))
block_holder.requirements = new_table
def test_delete_dep_table(self):
original_deptable = BlockVersionTable()
brl_block = BRLBlock('dummy/dummy/block/master')
block_version = BlockVersion(brl_block, 0)
self.db.set_dep_table(block_version, original_deptable)
ID = encode_serialized_value(block_version.serialize())
self.db.delete(ID, DEP_TABLES)
self.assertRaises(NotInStoreException, self.db.get_dep_table,
block_version)
def __init__(self, resources, tables):
""" Input: {version: resource}
"""
self._resources = {} # {Reference: Resource}
for version, resource in resources:
self._resources[Reference(version, resource.cell_name)] = resource
self._tables = {}
for version, dep_versions in tables.iteritems():
self._tables[version] = BlockVersionTable(dep_versions)
def __init__(self, text):
""" Create memory representation (parsing) of biicode block configuration file
param block_name: (BlockName) the block name this conf file lives in
param text: (str) the configuration text (normalized to LF only)
"""
self._template = text is None
self._text = text if text is not None else template
assert '\r' not in self._text
# Configuration fields
self.parent = None
self.requirements = BlockVersionTable()
self.dependencies = []
self.paths = []
self.mains = [] # Can be sets
self.includes = []
self.data = [] # Can be sets
self.tests = []
self._parse()
def __init__(self, policy=None):
self.unresolved = set() # Unresolved declarations to be found
self.block_names = set() # Current hive src blocks, to forbid cycles
self.existing = ReferencedDependencies() # Current resolved deps
self.existing_common_table = BlockVersionTable()
self.policy = policy
self.find = True # To find for UNRESOLVED
self.update = False # To activate UPDATES
self.downgrade = False # To activate DOWNGRADES
self.modify = False # Allow changing deps to non-descendant branches
def test_time_base_map(self):
t = TimeBaseMap()
mvt = BlockVersionTable([BlockVersion.loads("user/block: 3")])
t.append(0, mvt)
t.append(1, mvt)
s = serialize(t)
#print "TIME BASE:"+str(s)
t2 = TimeBaseMapDeserializer(BlockVersionTable).deserialize(s)
self.assertEqual(t, t2)
def test_block_version_table(self):
mv = BlockVersionTable()
mv.add_version(BlockVersion('user/user/block/master', 1))
mv.add_version(BlockVersion('user/user/block/master', 1))
mv.add_version(BlockVersion('user/user/block/master', 1))
s = mv.serialize()
mv2 = BlockVersionTable.deserialize(s)
self.assertEqual(mv, mv2)
def deserialize(data):
'''From dictionary to object FinderRequest'''
ret = FinderRequest()
ret.block_names = SetDeserializer(BlockName).deserialize(data[FinderRequest.SERIAL_TRACKING_KEY])
ret.existing = ReferencedDependencies.deserialize(data[FinderRequest.SERIAL_EXISTING_KEY])
ret.unresolved = SetDeserializer(Declaration).deserialize(data[FinderRequest.SERIAL_UNRESOLVED_KEY])
ret.policy = Policy.deserialize(data[FinderRequest.SERIAL_POLICY])
criteria = data[FinderRequest.SERIAL_CRITERIA]
ret.find, ret.update, ret.downgrade, ret.modify = criteria
ret.existing_common_table = BlockVersionTable.deserialize(data[FinderRequest.SERIAL_DEP_COMMON_TABLE])
return ret
def test_get_version_by_tag(self):
brl_block = BRLBlock('%s/%s/TestBlock/master' % (self.testUser.ID, self.testUser.ID))
publish_request = PublishRequest(BlockVersion(brl_block, -1))
publish_request.tag = STABLE
publish_request.versiontag = 'mytag'
publish_request.cells.append(SimpleCell(brl_block.block_name + 'r1.h'))
publish_request.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
publish_request.deptable = BlockVersionTable()
self.service.publish(publish_request)
publish_request = PublishRequest(BlockVersion(brl_block, 0, 'mytag'))
publish_request.tag = STABLE
publish_request.versiontag = 'mytag'
publish_request.cells.append(SimpleCell(brl_block.block_name + 'r12.h'))
publish_request.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
publish_request.deptable = BlockVersionTable()
self.service.publish(publish_request)
block_version = self.service.get_version_by_tag(brl_block, 'mytag')
self.assertEquals(1, block_version.time)
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_publish_concurrent_modification(self):
self.store = TestingMemServerStore()
brl = 'user'
self.store.create_user(User(brl))
self.service = BiiService(self.store, brl)
self.brl_block = BRLBlock('user/user/block/master')
request = PublishRequest(BlockVersion(self.brl_block, -1))
request.cells.append(SimpleCell('user/block/r1.h'))
request.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
request.deptable = BlockVersionTable()
request.tag = DEV
self.service.publish(request)
'''Branch 1 (from master) creates new resource'''
self.brl_block1 = BRLBlock('user/user/block/branch1')
request = PublishRequest(BlockVersion(self.brl_block, 0))
request.cells.append(SimpleCell('user/block/r2.h'))
request.contents['r2.h'] = Content(id_=None, load=Blob('adios'))
request.deptable = BlockVersionTable()
with self.assertRaisesRegexp(BiiException, 'Concurrent modification'):
self.service.publish(request)
def _parse(self):
sections = self._find_sections()
parent = sections.get('parent')
if parent:
self.parent = parent_loads(parent.content, parent.line)
self._old_parent = self.parent
reqs = sections.get('requirements')
if reqs:
self.requirements = BlockVersionTable.loads(
reqs.content, reqs.line)
self._old_requirements = self.requirements.copy()
# self.include_mapping = IncludesMapping.loads(includes)
deps = sections.get('dependencies')
if deps:
self.dependencies = parse_deps_conf(deps.content, deps.line)
hooks = sections.get('hooks')
if hooks:
hooks_deps = parse_deps_conf(hooks.content, hooks.line)
self.dependencies.extend(hooks_deps)
mains = sections.get('mains')
if mains:
self.mains = parse_mains_conf(mains.content, mains.line)
tests = sections.get('tests')
if tests:
def test_line_parser(line):
self.tests.append(line)
parse(tests.content, test_line_parser, tests.line)
paths = sections.get('paths')
if paths:
def path_line_parser(line):
self.paths.append(line)
parse(paths.content, path_line_parser, paths.line)
data = sections.get('data')
if data:
self.data = parse_deps_conf(data.content, data.line)
includes = sections.get('includes')
if includes:
self.includes = IncludesMapping.loads(includes.content,
includes.line)
def general_load_test(self):
block_name = BlockName("user/block")
version = BlockVersion(block_name.default_block(), 0)
config = BiiConfig(general)
self.assertEqual(config.parent, version)
v1 = BlockVersion.loads("user2/depblock2(track): 3 @tag")
v2 = BlockVersion.loads("user/depblock1(master): 3")
self.assertEqual(config.requirements, BlockVersionTable([v1, v2]))
self.assert_bii_equal(config.mains, [
EntryPointConfiguration("main.cpp", False),
EntryPointConfiguration("main2.cpp", True),
])
self.assert_bii_equal(config.tests, ["test.cpp", "test1.cpp"])
self.assertEqual(config.paths, ["/", "include"])
def _publishIndependentCell(self,
block_cell_name,
nVersions=10,
version_tag=None):
'''publishes nVersions of each cell'''
block_brl = self._getBlockBRL(block_cell_name)
for v in range(0, nVersions):
r = SimpleCell(block_cell_name)
pack = PublishRequest(BlockVersion(block_brl, v - 1))
pack.tag = version_tag if version_tag is not None else STABLE
pack.cells.append(r)
pack.deptable = BlockVersionTable()
pack.contents[r.name.cell_name] = Content(
id_=None, load=Blob('hola {0}'.format(v)))
self.service.publish(pack)
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(
data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(
data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(
data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def _parse(self):
sections = self._find_sections()
parent = sections.get('parent')
if parent:
self.parent = parent_loads(parent.content, parent.line)
self._old_parent = self.parent
reqs = sections.get('requirements')
if reqs:
self.requirements = BlockVersionTable.loads(reqs.content, reqs.line)
self._old_requirements = self.requirements.copy()
# self.include_mapping = IncludesMapping.loads(includes)
deps = sections.get('dependencies')
if deps:
self.dependencies = parse_deps_conf(deps.content, deps.line)
hooks = sections.get('hooks')
if hooks:
hooks_deps = parse_deps_conf(hooks.content, hooks.line)
self.dependencies.extend(hooks_deps)
mains = sections.get('mains')
if mains:
self.mains = parse_mains_conf(mains.content, mains.line)
tests = sections.get('tests')
if tests:
def test_line_parser(line):
self.tests.append(line)
parse(tests.content, test_line_parser, tests.line)
paths = sections.get('paths')
if paths:
def path_line_parser(line):
self.paths.append(line)
parse(paths.content, path_line_parser, paths.line)
data = sections.get('data')
if data:
self.data = parse_deps_conf(data.content, data.line)
includes = sections.get('includes')
if includes:
self.includes = IncludesMapping.loads(includes.content, includes.line)
def test_publish_rejected(self):
store = Mock(MongoServerStore)
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (Mock(), Mock())
store.read_block.return_value = block
brl = BRLBlock('user/user/block/branch')
p = PublishService(store, 'authUser')
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
with self.assertRaises(ForbiddenException):
p.publish(pack)
def __init__(self, text):
""" Create memory representation (parsing) of biicode block configuration file
param block_name: (BlockName) the block name this conf file lives in
param text: (str) the configuration text (normalized to LF only)
"""
self._template = text is None
self._text = text if text is not None else template
assert '\r' not in self._text
# Configuration fields
self.parent = None
self.requirements = BlockVersionTable()
self.dependencies = []
self.paths = []
self.mains = [] # Can be sets
self.includes = []
self.data = [] # Can be sets
self.tests = []
self._parse()
def deserialize(data):
'''From dictionary to object Publish Pack'''
pp = PublishRequest()
pp.parent = BlockVersion.deserialize(data[PublishRequest.SERIAL_TRACKED_KEY])
pp.parent_time = data[PublishRequest.SERIAL_PARENT_DATETIME]
pp.tag = VersionTag.deserialize(data[PublishRequest.SERIAL_TAG_KEY])
pp.msg = data[PublishRequest.SERIAL_MSG_KEY]
# Backward client compatibility
pp.versiontag = data.get(PublishRequest.SERIAL_VTAG_KEY, None)
pp.deptable = BlockVersionTable.deserialize(data[PublishRequest.SERIAL_DEP_TABLE])
pp.cells = ListDeserializer(CellDeserializer(BlockCellName)).\
deserialize(data[PublishRequest.SERIAL_CELLS_KEY])
pp.deleted = ListDeserializer(CellName).\
deserialize(data[PublishRequest.SERIAL_DELETED_KEY])
pp.renames = Renames.deserialize(data[PublishRequest.SERIAL_RENAMES_KEY])
pp.contents = DictDeserializer(CellName, ContentDeserializer(BlockCellName)).\
deserialize(data[PublishRequest.SERIAL_CONTENTS_KEY])
pp.contents_ids = DictDeserializer(CellName, ID).\
deserialize(data[PublishRequest.SERIAL_CONTENTS_ID_KEY])
# Backward client compatibility
pp.origin = OriginInfo.deserialize(data.get(PublishRequest.SERIAL_ORIGIN_INFO, None))
return pp
class BiiConfig(object):
sections = ['parent', 'requirements', 'paths', 'data', 'hooks', 'dependencies', 'includes',
'mains', 'tests']
def __init__(self, text):
""" Create memory representation (parsing) of biicode block configuration file
param block_name: (BlockName) the block name this conf file lives in
param text: (str) the configuration text (normalized to LF only)
"""
self._template = text is None
self._text = text if text is not None else template
assert '\r' not in self._text
# Configuration fields
self.parent = None
self.requirements = BlockVersionTable()
self.dependencies = []
self.paths = []
self.mains = [] # Can be sets
self.includes = []
self.data = [] # Can be sets
self.tests = []
self._parse()
def changed(self, other):
""" check if something different to parent, changed between 2 config files.
Necessary to discard changing parent as the only change in a block, e.g.
while checking empty (no-changes) publications
"""
return (self.requirements != other.requirements or
self.mains != other.mains or
self.paths != other.paths or
self.dependencies != other.dependencies or
self.data != other.data or
self.includes != other.includes or
self.tests != other.tests)
def _find_sections(self):
pattern = re.compile("^\[([a-z]{2,50})\]")
current_lines = []
sections = {}
Section = namedtuple("Section", "line content headline")
parsed_sections = set()
for i, line in enumerate(self._text.splitlines()):
m = pattern.match(line)
if m:
group = m.group(1)
if group not in BiiConfig.sections:
raise ConfigurationFileError("%d: Unrecognized section '%s'" % (i + 1, group))
if group in parsed_sections:
raise ConfigurationFileError("%d: Duplicated section '%s'" % (i + 1, group))
parsed_sections.add(group)
current_lines = []
sections[group] = Section(i + 1, current_lines, line)
else:
current_lines.append(line)
return {name: Section(line, '\n'.join(lines), headline)
for name, (line, lines, headline) in sections.iteritems()}
def _parse(self):
sections = self._find_sections()
parent = sections.get('parent')
if parent:
self.parent = parent_loads(parent.content, parent.line)
self._old_parent = self.parent
reqs = sections.get('requirements')
if reqs:
self.requirements = BlockVersionTable.loads(reqs.content, reqs.line)
self._old_requirements = self.requirements.copy()
# self.include_mapping = IncludesMapping.loads(includes)
deps = sections.get('dependencies')
if deps:
self.dependencies = parse_deps_conf(deps.content, deps.line)
hooks = sections.get('hooks')
if hooks:
hooks_deps = parse_deps_conf(hooks.content, hooks.line)
self.dependencies.extend(hooks_deps)
mains = sections.get('mains')
if mains:
self.mains = parse_mains_conf(mains.content, mains.line)
tests = sections.get('tests')
if tests:
def test_line_parser(line):
self.tests.append(line)
parse(tests.content, test_line_parser, tests.line)
paths = sections.get('paths')
if paths:
def path_line_parser(line):
self.paths.append(line)
parse(paths.content, path_line_parser, paths.line)
data = sections.get('data')
if data:
self.data = parse_deps_conf(data.content, data.line)
includes = sections.get('includes')
if includes:
self.includes = IncludesMapping.loads(includes.content, includes.line)
def _dump_requirements(self, sections):
if self.requirements != self._old_requirements:
assert all(v.time is not None for v in self.requirements.itervalues())
new_reqs = ["\t %s" % v.to_pretty() for v in sorted(self.requirements.itervalues())]
old_reqs = sections.get("requirements")
if old_reqs:
new_content = "%s\n%s\n" % (old_reqs.headline, '\n'.join(new_reqs))
old_content = "%s\n%s" % (old_reqs.headline, old_reqs.content)
self._text = self._text.replace(old_content, new_content)
else:
self._text = self._text + "\n[requirements]\n" + '\n'.join(new_reqs)
self._old_requirements = self.requirements.copy()
assert isinstance(self._old_requirements, BlockVersionTable)
return True
def _dump_parent(self, sections):
if self.parent != self._old_parent:
new_parent = "\t" + self.parent.to_pretty()
old = sections.get("parent")
if old:
new_content = "%s\n%s" % (old.headline, new_parent)
old_content = "%s\n%s" % (old.headline, old.content)
self._text = self._text.replace(old_content, new_content)
else:
self._text = self._text + "\n[parent]\n" + new_parent
self._old_parent = self.parent
return True
def dumps(self):
"""
return modified text (str) of configuration file or None if it didnt change
"""
sections = self._find_sections()
req_mod = self._dump_requirements(sections)
par_mod = self._dump_parent(sections)
if req_mod or par_mod or self._template:
self._template = False
return self._text