def _publishDependentCell(self, block_id):
low, high = self._biigraph._get_range_block(block_id)
count = 0
d = self.lbcn
bcn = d[block_id]
brl = self._getBlockBRL(bcn)
for row in range(low, high + 1):
r = SimpleCell(d[block_id])
r.root = ID((0, bcn.block_name[-1:], 0))
deps = DependencySet()
dep_table = BlockVersionTable()
for block in range(block_id):
time = self._biigraph._get_dep_elem_offset(row, block)
if time != -1:
deps.add_implicit(d[block])
dep_table[d[block].block_name] = \
BlockVersion(self._getBlockBRL(d[block]), time)
r.dependencies = deps
pack = PublishRequest(BlockVersion(brl, count - 1))
pack.tag = STABLE
count += 1
pack.cells.append(r)
pack.contents[r.name.cell_name] = Content(
id_=None, load=Blob('hola {0}'.format(r.name.cell_name)))
pack.deptable = dep_table
self.service.publish(pack)
def test_publish_dev_with_tag(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False))
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (['mock_id'], [], [], [])
block.deltas = []
ensure = Security('authUser', store)
ensure.check_read_block = Mock(return_value=True)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_publish_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.deptable = BlockVersionTable()
p.publish(pack)
def publish(self, brl_block, resource_info, version_tag=STABLE, dep_versions=None):
""" make a simple publication of a single file named block_cell_name to block
brl_block. If block does not exist, it creates it. It mantains coherence of root Ids
for the find checks
param resource_info: {cell_name: (blob, explicits)}
"""
try:
block = self.store.read_block(brl_block)
version = BlockVersion(brl_block, len(block.deltas) - 1)
except NotInStoreException:
version = BlockVersion(brl_block, -1)
publish_request = PublishRequest(version)
publish_request.tag = version_tag
block_name = brl_block.block_name
for cell_name, (blob, dependencies) in resource_info.iteritems():
if dependencies is not None:
cell = SimpleCell(block_name + cell_name)
if isinstance(dependencies, DependencySet):
cell.dependencies = dependencies
else:
cell.dependencies.explicit.update([BlockCellName(d) for d in dependencies])
publish_request.cells.append(cell)
if blob is not None:
blob = Blob(blob) if isinstance(blob, str) else blob
publish_request.contents[cell_name] = Content(id_=None, load=blob)
if isinstance(dep_versions, BlockVersion):
dep_versions = [dep_versions]
publish_request.deptable = BlockVersionTable(dep_versions)
self.service.publish(publish_request)
def _publishDependentCell(self, block_id):
low, high = self._biigraph._get_range_block(block_id)
count = 0
d = self.lbcn
bcn = d[block_id]
brl = self._getBlockBRL(bcn)
for row in range(low, high + 1):
r = SimpleCell(d[block_id])
r.root = ID((0, bcn.block_name[-1:], 0))
deps = DependencySet()
dep_table = BlockVersionTable()
for block in range(block_id):
time = self._biigraph._get_dep_elem_offset(row, block)
if time != -1:
deps.add_implicit(d[block])
dep_table[d[block].block_name] = \
BlockVersion(self._getBlockBRL(d[block]), time)
r.dependencies = deps
pack = PublishRequest(BlockVersion(brl, count - 1))
pack.tag = STABLE
count += 1
pack.cells.append(r)
pack.contents[r.name.cell_name] = Content(id_=None, load=Blob('hola {0}'.
format(r.name.cell_name)))
pack.deptable = dep_table
self.service.publish(pack)
def test_publish_dev_with_tag(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(
return_value=ElementPermissions(brl, private=False))
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (['mock_id'], [], [], [])
block.deltas = []
ensure = Security('authUser', store)
ensure.check_read_block = Mock(return_value=True)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_publish_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.deptable = BlockVersionTable()
p.publish(pack)
def test_publish(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
# moduleID=BlockID(UserID(123),456)
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(return_value=ElementPermissions(brl, private=False))
user = User("owner")
user.numeric_id = 1
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.last_version.return_value = Mock(BlockVersion)
block.add_publication.return_value = (Mock(list), Mock(list), Mock(list), Mock(list))
block.deltas = []
ensure = Security('authUser', store)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_read_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r2.h'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r3.h'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
p.publish(pack)
block.add_publication.assert_called_once_with(pack, p.auth_user)
store.update_block.assert_called_once_with(block)
self.assertEqual(1, store.create_published_cells.call_count)
self.assertEqual(1, store.create_published_contents.call_count)
# Check sizes
self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3
# Publish again, see the size incremented
pack._bytes = None # Lazy computed
p.publish(pack)
self.assertEquals(user.blocks_bytes, 24) # 24 bytes: "hola" * 3 * 2 publications
# Max size exceeded for user
user.max_workspace_size = 25
self.assertRaises(ForbiddenException, p.publish, pack)
# Try to publish only 1 byte
pack._bytes = None # Lazy computed
pack.cells = []
pack.contents = {}
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('h'))
p.publish(pack)
def test_get_version_by_tag(self):
brl_block = BRLBlock('%s/%s/TestBlock/master' % (self.testUser.ID, self.testUser.ID))
publish_request = PublishRequest(BlockVersion(brl_block, -1))
publish_request.tag = STABLE
publish_request.versiontag = 'mytag'
publish_request.cells.append(SimpleCell(brl_block.block_name + 'r1.h'))
publish_request.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
publish_request.deptable = BlockVersionTable()
self.service.publish(publish_request)
publish_request = PublishRequest(BlockVersion(brl_block, 0, 'mytag'))
publish_request.tag = STABLE
publish_request.versiontag = 'mytag'
publish_request.cells.append(SimpleCell(brl_block.block_name + 'r12.h'))
publish_request.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
publish_request.deptable = BlockVersionTable()
self.service.publish(publish_request)
block_version = self.service.get_version_by_tag(brl_block, 'mytag')
self.assertEquals(1, block_version.time)
def _publishIndependentCell(self, block_cell_name, nVersions=10,
version_tag=None):
'''publishes nVersions of each cell'''
block_brl = self._getBlockBRL(block_cell_name)
for v in range(0, nVersions):
r = SimpleCell(block_cell_name)
pack = PublishRequest(BlockVersion(block_brl, v - 1))
pack.tag = version_tag if version_tag is not None else STABLE
pack.cells.append(r)
pack.deptable = BlockVersionTable()
pack.contents[r.name.cell_name] = Content(id_=None, load=Blob('hola {0}'.format(v)))
self.service.publish(pack)
def test_publish_concurrent_modification(self):
self.store = TestingMemServerStore()
brl = 'user'
self.store.create_user(User(brl))
self.service = BiiService(self.store, brl)
self.brl_block = BRLBlock('user/user/block/master')
request = PublishRequest(BlockVersion(self.brl_block, -1))
request.cells.append(SimpleCell('user/block/r1.h'))
request.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
request.deptable = BlockVersionTable()
request.tag = DEV
self.service.publish(request)
'''Branch 1 (from master) creates new resource'''
self.brl_block1 = BRLBlock('user/user/block/branch1')
request = PublishRequest(BlockVersion(self.brl_block, 0))
request.cells.append(SimpleCell('user/block/r2.h'))
request.contents['r2.h'] = Content(id_=None, load=Blob('adios'))
request.deptable = BlockVersionTable()
with self.assertRaisesRegexp(BiiException, 'Concurrent modification'):
self.service.publish(request)
def test_publish_concurrent_modification(self):
self.store = TestingMemServerStore()
brl = 'user'
self.store.create_user(User(brl))
self.service = BiiService(self.store, brl)
self.brl_block = BRLBlock('user/user/block/master')
request = PublishRequest(BlockVersion(self.brl_block, -1))
request.cells.append(SimpleCell('user/block/r1.h'))
request.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
request.deptable = BlockVersionTable()
request.tag = DEV
self.service.publish(request)
'''Branch 1 (from master) creates new resource'''
self.brl_block1 = BRLBlock('user/user/block/branch1')
request = PublishRequest(BlockVersion(self.brl_block, 0))
request.cells.append(SimpleCell('user/block/r2.h'))
request.contents['r2.h'] = Content(id_=None, load=Blob('adios'))
request.deptable = BlockVersionTable()
with self.assertRaisesRegexp(BiiException, 'Concurrent modification'):
self.service.publish(request)
def _publishIndependentCell(self,
block_cell_name,
nVersions=10,
version_tag=None):
'''publishes nVersions of each cell'''
block_brl = self._getBlockBRL(block_cell_name)
for v in range(0, nVersions):
r = SimpleCell(block_cell_name)
pack = PublishRequest(BlockVersion(block_brl, v - 1))
pack.tag = version_tag if version_tag is not None else STABLE
pack.cells.append(r)
pack.deptable = BlockVersionTable()
pack.contents[r.name.cell_name] = Content(
id_=None, load=Blob('hola {0}'.format(v)))
self.service.publish(pack)
def test_publish_rejected(self):
store = Mock(MongoServerStore)
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (Mock(), Mock())
store.read_block.return_value = block
brl = BRLBlock('user/user/block/branch')
p = PublishService(store, 'authUser')
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
with self.assertRaises(ForbiddenException):
p.publish(pack)
def test_publish_rejected(self):
store = Mock(MongoServerStore)
user = Mock()
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.add_publication.return_value = (Mock(), Mock())
store.read_block.return_value = block
brl = BRLBlock('user/user/block/branch')
p = PublishService(store, 'authUser')
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
with self.assertRaises(ForbiddenException):
p.publish(pack)
def build_publish_request(biiapi, hive_holder, block_name, tag, msg,
versiontag, origin, biiout):
block_name, dep_block_names = _check_input(hive_holder, block_name)
_check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout)
block_holder = hive_holder[block_name]
if not changevalidator.check_block_size(block_holder, biiout):
raise PublishException("Block is too large to be published")
parent = block_holder.parent
_check_possible(parent, biiapi, biiout)
if parent.time != -1: # Update
remote_block_holder = biiapi.get_block_holder(parent)
base_resources = remote_block_holder.resources
parent_delta_info = biiapi.get_version_delta_info(parent)
else: # New block
base_resources = None
parent_delta_info = None
remote_block_holder = None
changes = compare(base_resources, block_holder.resources)
if not block_changed(changes, block_holder, remote_block_holder):
if parent_delta_info and tag > parent_delta_info.tag:
biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag))
changes.modified.pop(BIICODE_FILE, None)
else:
raise UpToDatePublishException("Up to date, nothing to publish")
changes.deduce_renames()
request = PublishRequest()
request.parent = parent
request.changes = changes
if parent_delta_info:
request.parent_time = parent_delta_info.date
assert all(bv.time is not None for bv in block_holder.requirements.itervalues())
request.deptable = block_holder.requirements
request.tag = tag
request.msg = msg
request.versiontag = versiontag
request.origin = origin
return request
def build_publish_request(biiapi, hive_holder, block_name, tag, msg,
versiontag, origin, biiout):
block_name, dep_block_names = _check_input(hive_holder, block_name)
_check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout)
block_holder = hive_holder[block_name]
if not changevalidator.check_block_size(block_holder, biiout):
raise PublishException("Block is too large to be published")
parent = block_holder.parent
_check_possible(parent, biiapi, biiout)
if parent.time != -1: # Update
remote_block_holder = biiapi.get_block_holder(parent)
base_resources = remote_block_holder.resources
parent_delta_info = biiapi.get_version_delta_info(parent)
else: # New block
base_resources = None
parent_delta_info = None
remote_block_holder = None
changes = compare(base_resources, block_holder.resources)
if not block_changed(changes, block_holder, remote_block_holder):
if parent_delta_info and tag > parent_delta_info.tag:
biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag))
changes.modified.pop(BIICODE_FILE, None)
else:
raise UpToDatePublishException("Up to date, nothing to publish")
changes.deduce_renames()
request = PublishRequest()
request.parent = parent
request.changes = changes
if parent_delta_info:
request.parent_time = parent_delta_info.date
assert all(bv.time is not None for bv in block_holder.requirements.itervalues())
request.deptable = block_holder.requirements
request.tag = tag
request.msg = msg
request.versiontag = versiontag
request.origin = origin
return request
def setUp(self):
self.conn.drop_database(self._testMethodName)
self.store = MongoServerStore(self.conn, self._testMethodName)
self.mother = ModelCreator(self.store)
self.user = self.mother.make_test_user()
self.p = PublishService(self.store, self.user.ID)
self.brl = BRLBlock('%s/%s/block/master' % (self.user.ID, self.user.ID))
pack = PublishRequest(BlockVersion(self.brl, -1))
pack.cells.append(SimpleCell('%s/block/r1.h' % self.user.ID))
pack.cells.append(SimpleCell('%s/block/r2.h' % self.user.ID))
pack.cells.append(SimpleCell('%s/block/r3.h' % self.user.ID))
pack.contents['r1.h'] = Content(id_=None, load=Blob('r1'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('r2'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('r3'))
pack.deptable = BlockVersionTable()
self.pack = pack
self.cell_collection = self.store.db[GenericServerStore.PUBLISHED_CELL_ST]
self.content_collection = self.store.db[GenericServerStore.PUBLISHED_CONTENT_ST]
self.block_collection = self.store.db[GenericServerStore.BLOCK_ST]
def publish(self,
brl_block,
resource_info,
version_tag=STABLE,
dep_versions=None):
""" make a simple publication of a single file named block_cell_name to block
brl_block. If block does not exist, it creates it. It mantains coherence of root Ids
for the find checks
param resource_info: {cell_name: (blob, explicits)}
"""
try:
block = self.store.read_block(brl_block)
version = BlockVersion(brl_block, len(block.deltas) - 1)
except NotInStoreException:
version = BlockVersion(brl_block, -1)
publish_request = PublishRequest(version)
publish_request.tag = version_tag
block_name = brl_block.block_name
for cell_name, (blob, dependencies) in resource_info.iteritems():
if dependencies is not None:
cell = SimpleCell(block_name + cell_name)
if isinstance(dependencies, DependencySet):
cell.dependencies = dependencies
else:
cell.dependencies.explicit.update(
[BlockCellName(d) for d in dependencies])
publish_request.cells.append(cell)
if blob is not None:
blob = Blob(blob) if isinstance(blob, str) else blob
publish_request.contents[cell_name] = Content(id_=None,
load=blob)
if isinstance(dep_versions, BlockVersion):
dep_versions = [dep_versions]
publish_request.deptable = BlockVersionTable(dep_versions)
self.service.publish(publish_request)
def setUp(self):
self.conn.drop_database(self._testMethodName)
self.store = MongoServerStore(self.conn, self._testMethodName)
self.mother = ModelCreator(self.store)
self.user = self.mother.make_test_user()
self.p = PublishService(self.store, self.user.ID)
self.brl = BRLBlock('%s/%s/block/master' %
(self.user.ID, self.user.ID))
pack = PublishRequest(BlockVersion(self.brl, -1))
pack.cells.append(SimpleCell('%s/block/r1.h' % self.user.ID))
pack.cells.append(SimpleCell('%s/block/r2.h' % self.user.ID))
pack.cells.append(SimpleCell('%s/block/r3.h' % self.user.ID))
pack.contents['r1.h'] = Content(id_=None, load=Blob('r1'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('r2'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('r3'))
pack.deptable = BlockVersionTable()
self.pack = pack
self.cell_collection = self.store.db[
GenericServerStore.PUBLISHED_CELL_ST]
self.content_collection = self.store.db[
GenericServerStore.PUBLISHED_CONTENT_ST]
self.block_collection = self.store.db[GenericServerStore.BLOCK_ST]
def test_publish(self, enqueuer):
brl = BRLBlock('owner/user/block/branch')
# moduleID=BlockID(UserID(123),456)
store = Mock(MongoServerStore)
store.read_block_permissions = Mock(
return_value=ElementPermissions(brl, private=False))
user = User("owner")
user.numeric_id = 1
user.blocks = {}
store.read_user = Mock(return_value=user)
block = Mock(Block)
block.last_version.return_value = Mock(BlockVersion)
block.add_publication.return_value = (Mock(list), Mock(list),
Mock(list), Mock(list))
block.deltas = []
ensure = Security('authUser', store)
ensure.check_create_block = Mock(return_value=True)
ensure.check_write_block = Mock(return_value=True)
ensure.check_read_block = Mock(return_value=True)
store.read_block.return_value = block
store.read_published_cells.return_value = {}
p = PublishService(store, 'authUser')
p.security = ensure
pack = PublishRequest(BlockVersion(brl, -1))
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r2.h'))
pack.contents['r2.h'] = Content(id_=None, load=Blob('hola'))
pack.cells.append(SimpleCell('user/block/r3.h'))
pack.contents['r3.h'] = Content(id_=None, load=Blob('hola'))
pack.deptable = BlockVersionTable()
p.publish(pack)
block.add_publication.assert_called_once_with(pack, p.auth_user)
store.update_block.assert_called_once_with(block)
self.assertEqual(1, store.create_published_cells.call_count)
self.assertEqual(1, store.create_published_contents.call_count)
# Check sizes
self.assertEquals(user.blocks_bytes, 12) # 12 bytes "hola" * 3
# Publish again, see the size incremented
pack._bytes = None # Lazy computed
p.publish(pack)
self.assertEquals(user.blocks_bytes,
24) # 24 bytes: "hola" * 3 * 2 publications
# Max size exceeded for user
user.max_workspace_size = 25
self.assertRaises(ForbiddenException, p.publish, pack)
# Try to publish only 1 byte
pack._bytes = None # Lazy computed
pack.cells = []
pack.contents = {}
pack.cells.append(SimpleCell('user/block/r1.h'))
pack.contents['r1.h'] = Content(id_=None, load=Blob('h'))
p.publish(pack)