def merge(self,
base,
other,
common,
base_renames=None,
other_renames=None):
'''
Merges two blocks
Params:
base: dict { name: value }
other: dict { name: value }
common: dict { name: value }
base_renames: Renames object
other_renames: Renames object
Returns:
Dict CellName => Resource with merged contents
'''
if not base_renames:
base_renames = {}
if not other_renames:
other_renames = {}
base_changes = compare(common, base)
other_changes = compare(common, other)
base_changes.renames = base_renames
other_changes.renames = other_renames
ret = copy(common)
self._handle_deletion(base_changes, other_changes, ret)
self._handle_deletion(other_changes, base_changes, ret)
self._handle_creation(base_changes, other_changes, ret)
self._handle_creation(other_changes, base_changes, ret)
self._handle_modify(base_changes, other_changes, ret)
self._handle_modify(other_changes, base_changes, ret)
self._handle_renames(base_changes, other_changes, ret)
self._handle_renames(other_changes, base_changes, ret)
assert not base_changes.deleted, base_changes.deleted
assert not base_changes.created, base_changes.created
assert not base_changes.modified, base_changes.modified
assert not base_changes.renames, base_changes.renames
assert not other_changes.deleted, other_changes.deleted
assert not other_changes.created, other_changes.created
assert not other_changes.modified, other_changes.modified
assert not other_changes.renames, other_changes.renames
return ret
def _check_dep_blocks(biiapi, hive_holder, dep_blocks, biiout):
'''check that the dependent blocks have no modifications'''
modified = False
parents = {b.parent.block_name: b.parent for b in hive_holder.block_holders}
for block_name in dep_blocks:
edition_block_holder = hive_holder[block_name]
# Modify requirements (only in memory) for comparison _block_changed below
requirements = edition_block_holder.requirements
for parent_name, parent in parents.iteritems():
if parent_name in requirements:
requirements.add_version(parent)
# Now check parents
parent = edition_block_holder.parent
if parent.time == -1:
modified = True
biiout.error('Block %s has never been published. Publish it first' % (block_name))
else:
remote_block_holder = biiapi.get_block_holder(parent)
changes = compare(remote_block_holder.resources, edition_block_holder.resources)
if block_changed(changes, edition_block_holder, remote_block_holder):
modified = True
biiout.error('Block %s is modified. Publish it first' % (block_name))
if modified:
raise PublishException('There are modified blocks that must be published first')
def close(self, block_name, settings=None, force=False):
from biicode.common.edition.open_close import close_block
assert isinstance(block_name, BlockName)
if block_name not in self.hive_holder.blocks:
raise BiiException("Block %s is not in your project, you cannot close it"
% str(block_name))
parent = self.hive_holder[block_name].parent
if parent.time == -1:
raise BiiException("Block %s is not published, you cannot close it."
"\nIf you want to delete it, delete the folder in the filesystem."
% str(block_name))
remote_block_holder = self._biiapi.get_block_holder(parent)
from biicode.common.diffmerge.compare import compare
changes = compare(remote_block_holder.resources, self.hive_holder[block_name].resources)
if block_changed(changes, self.hive_holder[block_name], remote_block_holder) and not force:
raise BiiException("Block %s has unpublished changes.\n%s\n"
"Execute with --force to ignore them or publish it first."
% (str(block_name), str(changes)))
processor_changes = close_block(self.hive_holder, block_name)
blocks_process(self.hive_holder, processor_changes, self._biiout)
deps_process(self._biiapi, self.hive_holder, processor_changes, self._biiout, settings)
self._edition.save_hive_changes(self.hive_holder.hive, processor_changes)
def test_deduce_renames(self):
''' 2 is modified from 2 to 22
3 is deleted
5 is renamed to 6 (5 deleted, 6 created)
10 is created'''
base_resources = {CellName('1'): Int(1), CellName('2'): Int(2),
CellName('3'): Int(3), CellName('4'): Int(4),
CellName('5'): Int(5)}
other_resources = {CellName('1'): Int(1), CellName('2'): Int(22),
CellName('4'): Int(4), CellName('6'): Int(6),
CellName('10'): Int(10)}
#compute changes without renames
changes = compare(base_resources, other_resources)
self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted)
self.assertEqual({CellName('6'): Int(6), CellName('10'): 10}, changes.created)
self.assertEqual({CellName('2'): (2, 22)}, changes.modified)
self.assertEqual({}, changes.renames)
#deduce renames
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted)
self.assertEqual({CellName('6'): Int(6), CellName('10'): 10}, changes.created)
self.assertEqual({CellName('2'): (2, 22)}, changes.modified)
#but the renames field
self.assertEqual({CellName('5'): CellName('6')}, changes.renames)
def test_deduce_renames_multi_all_equal(self):
'''2 is deleted
3 is created with 2's contents
4 is created with 2's contents
2 is considered to be renamed to 4
'''
#FIXME: Conclusion is arbitrary. Last one to be processed with equal similarty degreee
# is the one choosen. We might have to inform the user about this
base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)}
other_resources = {
CellName('1'): Int(1),
CellName('3'): Int(2),
CellName('4'): Int(2)
}
#compute changes without renames
changes = compare(base_resources, other_resources)
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('2'): 2}, changes.deleted)
self.assertEqual({
CellName('3'): Int(2),
CellName('4'): 2
}, changes.created)
self.assertEqual({}, changes.modified)
self.assertEqual({CellName('2'): CellName('4')}, changes.renames)
def _check_dep_blocks(biiapi, hive_holder, dep_blocks, biiout):
'''check that the dependent blocks have no modifications'''
modified = False
parents = {b.parent.block_name: b.parent for b in hive_holder.block_holders}
for block_name in dep_blocks:
edition_block_holder = hive_holder[block_name]
# Modify requirements (only in memory) for comparison _block_changed below
requirements = edition_block_holder.requirements
for parent_name, parent in parents.iteritems():
if parent_name in requirements:
requirements.add_version(parent)
# Now check parents
parent = edition_block_holder.parent
if parent.time == -1:
modified = True
biiout.error('Block %s has never been published. Publish it first' % (block_name))
else:
remote_block_holder = biiapi.get_block_holder(parent)
changes = compare(remote_block_holder.resources, edition_block_holder.resources)
if block_changed(changes, edition_block_holder, remote_block_holder):
modified = True
biiout.error('Block %s is modified. Publish it first' % (block_name))
if modified:
raise PublishException('There are modified blocks that must be published first')
def update(block_holder, time, biiapi, biiout):
block_name = block_holder.block_name
parent = block_holder.parent
if parent is None:
raise BiiException('No parent info in block: %s' % block_name)
biiout.info("Updating %s" % parent.to_pretty())
if time is None:
block_info = biiapi.get_block_info(parent.block)
block_version = block_info.last_version
time = block_version.time
if time < parent.time:
raise BiiException('Impossible to update block %s to an older version.\n'
'Current version %d > Requested version %d'
% (parent.block.to_pretty(), parent.time, time))
# Tags for text merge
base_tag_name = 'current'
other_tag_name = 'v%d' % time
# Prepare merge info
base_resources = block_holder.resources
other_version = BlockVersion(parent.block, time)
common_holder = biiapi.get_block_holder(parent)
common_holder.parent = parent
common_holder.commit_config()
common_resources = common_holder.resources
if time == parent.time: # For updating with self parent
other_holder = common_holder
common_resources = base_resources # The common is the base, otherwise outdate
other_renames = {}
else:
other_holder = biiapi.get_block_holder(other_version)
other_holder.parent = parent
other_holder.commit_config()
other_renames = biiapi.get_renames(parent.block, parent.time, time)
other_resources = other_holder.resources
content_changes = compare(common_resources, base_resources)
content_changes.deduce_renames()
base_renames = content_changes.renames
# Merge resources
base_blobs = {name: content.load for name, (_, content) in base_resources.iteritems()
if content is not None}
other_blobs = {name: content.load for name, (_, content) in other_resources.iteritems()
if content is not None}
common_blobs = {name: content.load for name, (_, content) in common_resources.iteritems()
if content is not None}
merger = BlobsMerger(base_tag_name, other_tag_name, biiout)
result = merger.merge(base_blobs, other_blobs, common_blobs, base_renames, other_renames)
final_result = {name: blob.load for name, blob in result.iteritems()}
return final_result, other_version
def merge(self, base, other, common, base_renames=None, other_renames=None):
'''
Merges two blocks
Params:
base: dict { name: value }
other: dict { name: value }
common: dict { name: value }
base_renames: Renames object
other_renames: Renames object
Returns:
Dict CellName => Resource with merged contents
'''
if not base_renames:
base_renames = {}
if not other_renames:
other_renames = {}
base_changes = compare(common, base)
other_changes = compare(common, other)
base_changes.renames = base_renames
other_changes.renames = other_renames
ret = copy(common)
self._handle_deletion(base_changes, other_changes, ret)
self._handle_deletion(other_changes, base_changes, ret)
self._handle_creation(base_changes, other_changes, ret)
self._handle_creation(other_changes, base_changes, ret)
self._handle_modify(base_changes, other_changes, ret)
self._handle_modify(other_changes, base_changes, ret)
self._handle_renames(base_changes, other_changes, ret)
self._handle_renames(other_changes, base_changes, ret)
assert not base_changes.deleted, base_changes.deleted
assert not base_changes.created, base_changes.created
assert not base_changes.modified, base_changes.modified
assert not base_changes.renames, base_changes.renames
assert not other_changes.deleted, other_changes.deleted
assert not other_changes.created, other_changes.created
assert not other_changes.modified, other_changes.modified
assert not other_changes.renames, other_changes.renames
return ret
def test_delete_resource(self):
edition_resources = {}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(1, len(changes.deleted))
self.assertEqual(0, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(0, len(changes.modified))
self.assertEqual((self.r1, self.c1), changes.deleted['r1.h'])
def test_delete_resource(self):
edition_resources = {}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(1, len(changes.deleted))
self.assertEqual(0, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(0, len(changes.modified))
self.assertEqual((self.r1, self.c1), changes.deleted['r1.h'])
def test_diff_text(self):
base = {'f1': 'Hola que tal\nBien Gracias\n',
'f2': 'Hasta luego\n'}
other = {'f1': 'Hola que tal\nBien, Bien\n',
'f3': 'Hasta luego lucas\n'}
changes = compare(base, other)
diff = compute_diff(changes, text_unified_diff)
self.assertIn('-Hasta luego', diff.deleted['f2'])
self.assertIn('+Hasta luego lucas', diff.created['f3'])
self.assertIn('-Bien Gracias', diff.modified['f1'])
self.assertIn('+Bien, Bien', diff.modified['f1'])
def test_diff_text(self):
base = {'f1': 'Hola que tal\nBien Gracias\n', 'f2': 'Hasta luego\n'}
other = {
'f1': 'Hola que tal\nBien, Bien\n',
'f3': 'Hasta luego lucas\n'
}
changes = compare(base, other)
diff = compute_diff(changes, text_unified_diff)
self.assertIn('-Hasta luego', diff.deleted['f2'])
self.assertIn('+Hasta luego lucas', diff.created['f3'])
self.assertIn('-Bien Gracias', diff.modified['f1'])
self.assertIn('+Bien, Bien', diff.modified['f1'])
def test_modify_content(self):
'''just modify the content of a file'''
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('bye'))
edition_resources = {'r1.h': Resource(r1, content)}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(0, len(changes.deleted))
self.assertEqual(0, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(1, len(changes.modified))
self.assertEqual(Resource(r1, content), changes.modified['r1.h'].new)
def test_modify_content(self):
'''just modify the content of a file'''
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('bye'))
edition_resources = {'r1.h': Resource(r1, content)}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(0, len(changes.deleted))
self.assertEqual(0, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(1, len(changes.modified))
self.assertEqual(Resource(r1, content), changes.modified['r1.h'].new)
def test_create_resource(self):
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('hello'))
r2 = SimpleCell(self.brl_block.block_name + 'r2.h')
content2 = Content(id_=None, load=Blob('bye'))
edition_resources = {'r1.h': Resource(r1, content), 'r2.h': Resource(r2, content2)}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(0, len(changes.deleted))
self.assertEqual(1, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(0, len(changes.modified))
self.assertEqual(Resource(r2, content2), changes.created['r2.h'])
def test_modify_content_diff(self):
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('bye'))
edition_resources = {'r1.h': Resource(r1, content)}
changes = compare(self.last_version_resources, edition_resources)
diff = compute_diff(changes, resource_diff_function)
self.assertEqual(0, len(diff.deleted))
self.assertEqual(0, len(diff.created))
self.assertEqual(0, len(diff.renames))
self.assertEqual(1, len(diff.modified))
self.assertEqual('--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye',
diff.modified['r1.h'].content)
#print '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'
self.assertEqual(Resource(None, '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'),
diff.modified['r1.h'])
def test_deduce_renames_multi_different_values(self):
'''2 is deleted
3 is created with 3
4 is created with 4
2 is considered to be renamed to 3
'''
base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)}
other_resources = {CellName('1'): Int(1),
CellName('3'): Int(3), CellName('4'): Int(4)}
#compute changes without renames
changes = compare(base_resources, other_resources)
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('2'): 2}, changes.deleted)
self.assertEqual({CellName('3'): Int(3), CellName('4'): 4}, changes.created)
self.assertEqual({}, changes.modified)
self.assertEqual({CellName('2'): CellName('3')}, changes.renames)
def test_create_resource(self):
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('hello'))
r2 = SimpleCell(self.brl_block.block_name + 'r2.h')
content2 = Content(id_=None, load=Blob('bye'))
edition_resources = {
'r1.h': Resource(r1, content),
'r2.h': Resource(r2, content2)
}
changes = compare(self.last_version_resources, edition_resources)
self.assertEqual(0, len(changes.deleted))
self.assertEqual(1, len(changes.created))
self.assertEqual(0, len(changes.renames))
self.assertEqual(0, len(changes.modified))
self.assertEqual(Resource(r2, content2), changes.created['r2.h'])
def test_modify_content_diff(self):
r1 = SimpleCell(self.brl_block.block_name + 'r1.h')
content = Content(id_=None, load=Blob('bye'))
edition_resources = {'r1.h': Resource(r1, content)}
changes = compare(self.last_version_resources, edition_resources)
diff = compute_diff(changes, resource_diff_function)
self.assertEqual(0, len(diff.deleted))
self.assertEqual(0, len(diff.created))
self.assertEqual(0, len(diff.renames))
self.assertEqual(1, len(diff.modified))
self.assertEqual(
'--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye',
diff.modified['r1.h'].content)
#print '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'
self.assertEqual(
Resource(None,
'--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'),
diff.modified['r1.h'])
def test_deduce_renames_multi_all_equal(self):
'''2 is deleted
3 is created with 2's contents
4 is created with 2's contents
2 is considered to be renamed to 4
'''
#FIXME: Conclusion is arbitrary. Last one to be processed with equal similarty degreee
# is the one choosen. We might have to inform the user about this
base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)}
other_resources = {CellName('1'): Int(1),
CellName('3'): Int(2), CellName('4'): Int(2)}
#compute changes without renames
changes = compare(base_resources, other_resources)
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('2'): 2}, changes.deleted)
self.assertEqual({CellName('3'): Int(2), CellName('4'): 2}, changes.created)
self.assertEqual({}, changes.modified)
self.assertEqual({CellName('2'): CellName('4')}, changes.renames)
def build_publish_request(biiapi, hive_holder, block_name, tag, msg,
versiontag, origin, biiout):
block_name, dep_block_names = _check_input(hive_holder, block_name)
_check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout)
block_holder = hive_holder[block_name]
if not changevalidator.check_block_size(block_holder, biiout):
raise PublishException("Block is too large to be published")
parent = block_holder.parent
_check_possible(parent, biiapi, biiout)
if parent.time != -1: # Update
remote_block_holder = biiapi.get_block_holder(parent)
base_resources = remote_block_holder.resources
parent_delta_info = biiapi.get_version_delta_info(parent)
else: # New block
base_resources = None
parent_delta_info = None
remote_block_holder = None
changes = compare(base_resources, block_holder.resources)
if not block_changed(changes, block_holder, remote_block_holder):
if parent_delta_info and tag > parent_delta_info.tag:
biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag))
changes.modified.pop(BIICODE_FILE, None)
else:
raise UpToDatePublishException("Up to date, nothing to publish")
changes.deduce_renames()
request = PublishRequest()
request.parent = parent
request.changes = changes
if parent_delta_info:
request.parent_time = parent_delta_info.date
assert all(bv.time is not None for bv in block_holder.requirements.itervalues())
request.deptable = block_holder.requirements
request.tag = tag
request.msg = msg
request.versiontag = versiontag
request.origin = origin
return request
def test_deduce_renames(self):
''' 2 is modified from 2 to 22
3 is deleted
5 is renamed to 6 (5 deleted, 6 created)
10 is created'''
base_resources = {
CellName('1'): Int(1),
CellName('2'): Int(2),
CellName('3'): Int(3),
CellName('4'): Int(4),
CellName('5'): Int(5)
}
other_resources = {
CellName('1'): Int(1),
CellName('2'): Int(22),
CellName('4'): Int(4),
CellName('6'): Int(6),
CellName('10'): Int(10)
}
#compute changes without renames
changes = compare(base_resources, other_resources)
self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted)
self.assertEqual({
CellName('6'): Int(6),
CellName('10'): 10
}, changes.created)
self.assertEqual({CellName('2'): (2, 22)}, changes.modified)
self.assertEqual({}, changes.renames)
#deduce renames
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted)
self.assertEqual({
CellName('6'): Int(6),
CellName('10'): 10
}, changes.created)
self.assertEqual({CellName('2'): (2, 22)}, changes.modified)
#but the renames field
self.assertEqual({CellName('5'): CellName('6')}, changes.renames)
def build_publish_request(biiapi, hive_holder, block_name, tag, msg,
versiontag, origin, biiout):
block_name, dep_block_names = _check_input(hive_holder, block_name)
_check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout)
block_holder = hive_holder[block_name]
if not changevalidator.check_block_size(block_holder, biiout):
raise PublishException("Block is too large to be published")
parent = block_holder.parent
_check_possible(parent, biiapi, biiout)
if parent.time != -1: # Update
remote_block_holder = biiapi.get_block_holder(parent)
base_resources = remote_block_holder.resources
parent_delta_info = biiapi.get_version_delta_info(parent)
else: # New block
base_resources = None
parent_delta_info = None
remote_block_holder = None
changes = compare(base_resources, block_holder.resources)
if not block_changed(changes, block_holder, remote_block_holder):
if parent_delta_info and tag > parent_delta_info.tag:
biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag))
changes.modified.pop(BIICODE_FILE, None)
else:
raise UpToDatePublishException("Up to date, nothing to publish")
changes.deduce_renames()
request = PublishRequest()
request.parent = parent
request.changes = changes
if parent_delta_info:
request.parent_time = parent_delta_info.date
assert all(bv.time is not None for bv in block_holder.requirements.itervalues())
request.deptable = block_holder.requirements
request.tag = tag
request.msg = msg
request.versiontag = versiontag
request.origin = origin
return request
def test_deduce_renames_multi_different_values(self):
'''2 is deleted
3 is created with 3
4 is created with 4
2 is considered to be renamed to 3
'''
base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)}
other_resources = {
CellName('1'): Int(1),
CellName('3'): Int(3),
CellName('4'): Int(4)
}
#compute changes without renames
changes = compare(base_resources, other_resources)
changes.deduce_renames()
#nothing changes
self.assertEqual({CellName('2'): 2}, changes.deleted)
self.assertEqual({
CellName('3'): Int(3),
CellName('4'): 4
}, changes.created)
self.assertEqual({}, changes.modified)
self.assertEqual({CellName('2'): CellName('3')}, changes.renames)
def update(block_holder, time, biiapi, biiout):
block_name = block_holder.block_name
parent = block_holder.parent
if parent is None:
raise BiiException('No parent info in block: %s' % block_name)
biiout.info("Updating %s" % parent.to_pretty())
if time is None:
block_info = biiapi.get_block_info(parent.block)
block_version = block_info.last_version
time = block_version.time
if time < parent.time:
raise BiiException(
'Impossible to update block %s to an older version.\n'
'Current version %d > Requested version %d' %
(parent.block.to_pretty(), parent.time, time))
# Tags for text merge
base_tag_name = 'current'
other_tag_name = 'v%d' % time
# Prepare merge info
base_resources = block_holder.resources
other_version = BlockVersion(parent.block, time)
common_holder = biiapi.get_block_holder(parent)
common_holder.parent = parent
common_holder.commit_config()
common_resources = common_holder.resources
if time == parent.time: # For updating with self parent
other_holder = common_holder
common_resources = base_resources # The common is the base, otherwise outdate
other_renames = {}
else:
other_holder = biiapi.get_block_holder(other_version)
other_holder.parent = parent
other_holder.commit_config()
other_renames = biiapi.get_renames(parent.block, parent.time, time)
other_resources = other_holder.resources
content_changes = compare(common_resources, base_resources)
content_changes.deduce_renames()
base_renames = content_changes.renames
# Merge resources
base_blobs = {
name: content.load
for name, (_, content) in base_resources.iteritems()
if content is not None
}
other_blobs = {
name: content.load
for name, (_, content) in other_resources.iteritems()
if content is not None
}
common_blobs = {
name: content.load
for name, (_, content) in common_resources.iteritems()
if content is not None
}
merger = BlobsMerger(base_tag_name, other_tag_name, biiout)
result = merger.merge(base_blobs, other_blobs, common_blobs, base_renames,
other_renames)
final_result = {name: blob.load for name, blob in result.iteritems()}
return final_result, other_version