def merge(self, base, other, common, base_renames=None, other_renames=None): ''' Merges two blocks Params: base: dict { name: value } other: dict { name: value } common: dict { name: value } base_renames: Renames object other_renames: Renames object Returns: Dict CellName => Resource with merged contents ''' if not base_renames: base_renames = {} if not other_renames: other_renames = {} base_changes = compare(common, base) other_changes = compare(common, other) base_changes.renames = base_renames other_changes.renames = other_renames ret = copy(common) self._handle_deletion(base_changes, other_changes, ret) self._handle_deletion(other_changes, base_changes, ret) self._handle_creation(base_changes, other_changes, ret) self._handle_creation(other_changes, base_changes, ret) self._handle_modify(base_changes, other_changes, ret) self._handle_modify(other_changes, base_changes, ret) self._handle_renames(base_changes, other_changes, ret) self._handle_renames(other_changes, base_changes, ret) assert not base_changes.deleted, base_changes.deleted assert not base_changes.created, base_changes.created assert not base_changes.modified, base_changes.modified assert not base_changes.renames, base_changes.renames assert not other_changes.deleted, other_changes.deleted assert not other_changes.created, other_changes.created assert not other_changes.modified, other_changes.modified assert not other_changes.renames, other_changes.renames return ret
def _check_dep_blocks(biiapi, hive_holder, dep_blocks, biiout): '''check that the dependent blocks have no modifications''' modified = False parents = {b.parent.block_name: b.parent for b in hive_holder.block_holders} for block_name in dep_blocks: edition_block_holder = hive_holder[block_name] # Modify requirements (only in memory) for comparison _block_changed below requirements = edition_block_holder.requirements for parent_name, parent in parents.iteritems(): if parent_name in requirements: requirements.add_version(parent) # Now check parents parent = edition_block_holder.parent if parent.time == -1: modified = True biiout.error('Block %s has never been published. Publish it first' % (block_name)) else: remote_block_holder = biiapi.get_block_holder(parent) changes = compare(remote_block_holder.resources, edition_block_holder.resources) if block_changed(changes, edition_block_holder, remote_block_holder): modified = True biiout.error('Block %s is modified. Publish it first' % (block_name)) if modified: raise PublishException('There are modified blocks that must be published first')
def close(self, block_name, settings=None, force=False): from biicode.common.edition.open_close import close_block assert isinstance(block_name, BlockName) if block_name not in self.hive_holder.blocks: raise BiiException("Block %s is not in your project, you cannot close it" % str(block_name)) parent = self.hive_holder[block_name].parent if parent.time == -1: raise BiiException("Block %s is not published, you cannot close it." "\nIf you want to delete it, delete the folder in the filesystem." % str(block_name)) remote_block_holder = self._biiapi.get_block_holder(parent) from biicode.common.diffmerge.compare import compare changes = compare(remote_block_holder.resources, self.hive_holder[block_name].resources) if block_changed(changes, self.hive_holder[block_name], remote_block_holder) and not force: raise BiiException("Block %s has unpublished changes.\n%s\n" "Execute with --force to ignore them or publish it first." % (str(block_name), str(changes))) processor_changes = close_block(self.hive_holder, block_name) blocks_process(self.hive_holder, processor_changes, self._biiout) deps_process(self._biiapi, self.hive_holder, processor_changes, self._biiout, settings) self._edition.save_hive_changes(self.hive_holder.hive, processor_changes)
def test_deduce_renames(self): ''' 2 is modified from 2 to 22 3 is deleted 5 is renamed to 6 (5 deleted, 6 created) 10 is created''' base_resources = {CellName('1'): Int(1), CellName('2'): Int(2), CellName('3'): Int(3), CellName('4'): Int(4), CellName('5'): Int(5)} other_resources = {CellName('1'): Int(1), CellName('2'): Int(22), CellName('4'): Int(4), CellName('6'): Int(6), CellName('10'): Int(10)} #compute changes without renames changes = compare(base_resources, other_resources) self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted) self.assertEqual({CellName('6'): Int(6), CellName('10'): 10}, changes.created) self.assertEqual({CellName('2'): (2, 22)}, changes.modified) self.assertEqual({}, changes.renames) #deduce renames changes.deduce_renames() #nothing changes self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted) self.assertEqual({CellName('6'): Int(6), CellName('10'): 10}, changes.created) self.assertEqual({CellName('2'): (2, 22)}, changes.modified) #but the renames field self.assertEqual({CellName('5'): CellName('6')}, changes.renames)
def test_deduce_renames_multi_all_equal(self): '''2 is deleted 3 is created with 2's contents 4 is created with 2's contents 2 is considered to be renamed to 4 ''' #FIXME: Conclusion is arbitrary. Last one to be processed with equal similarty degreee # is the one choosen. We might have to inform the user about this base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)} other_resources = { CellName('1'): Int(1), CellName('3'): Int(2), CellName('4'): Int(2) } #compute changes without renames changes = compare(base_resources, other_resources) changes.deduce_renames() #nothing changes self.assertEqual({CellName('2'): 2}, changes.deleted) self.assertEqual({ CellName('3'): Int(2), CellName('4'): 2 }, changes.created) self.assertEqual({}, changes.modified) self.assertEqual({CellName('2'): CellName('4')}, changes.renames)
def _check_dep_blocks(biiapi, hive_holder, dep_blocks, biiout): '''check that the dependent blocks have no modifications''' modified = False parents = {b.parent.block_name: b.parent for b in hive_holder.block_holders} for block_name in dep_blocks: edition_block_holder = hive_holder[block_name] # Modify requirements (only in memory) for comparison _block_changed below requirements = edition_block_holder.requirements for parent_name, parent in parents.iteritems(): if parent_name in requirements: requirements.add_version(parent) # Now check parents parent = edition_block_holder.parent if parent.time == -1: modified = True biiout.error('Block %s has never been published. Publish it first' % (block_name)) else: remote_block_holder = biiapi.get_block_holder(parent) changes = compare(remote_block_holder.resources, edition_block_holder.resources) if block_changed(changes, edition_block_holder, remote_block_holder): modified = True biiout.error('Block %s is modified. Publish it first' % (block_name)) if modified: raise PublishException('There are modified blocks that must be published first')
def update(block_holder, time, biiapi, biiout): block_name = block_holder.block_name parent = block_holder.parent if parent is None: raise BiiException('No parent info in block: %s' % block_name) biiout.info("Updating %s" % parent.to_pretty()) if time is None: block_info = biiapi.get_block_info(parent.block) block_version = block_info.last_version time = block_version.time if time < parent.time: raise BiiException('Impossible to update block %s to an older version.\n' 'Current version %d > Requested version %d' % (parent.block.to_pretty(), parent.time, time)) # Tags for text merge base_tag_name = 'current' other_tag_name = 'v%d' % time # Prepare merge info base_resources = block_holder.resources other_version = BlockVersion(parent.block, time) common_holder = biiapi.get_block_holder(parent) common_holder.parent = parent common_holder.commit_config() common_resources = common_holder.resources if time == parent.time: # For updating with self parent other_holder = common_holder common_resources = base_resources # The common is the base, otherwise outdate other_renames = {} else: other_holder = biiapi.get_block_holder(other_version) other_holder.parent = parent other_holder.commit_config() other_renames = biiapi.get_renames(parent.block, parent.time, time) other_resources = other_holder.resources content_changes = compare(common_resources, base_resources) content_changes.deduce_renames() base_renames = content_changes.renames # Merge resources base_blobs = {name: content.load for name, (_, content) in base_resources.iteritems() if content is not None} other_blobs = {name: content.load for name, (_, content) in other_resources.iteritems() if content is not None} common_blobs = {name: content.load for name, (_, content) in common_resources.iteritems() if content is not None} merger = BlobsMerger(base_tag_name, other_tag_name, biiout) result = merger.merge(base_blobs, other_blobs, common_blobs, base_renames, other_renames) final_result = {name: blob.load for name, blob in result.iteritems()} return final_result, other_version
def merge(self, base, other, common, base_renames=None, other_renames=None): ''' Merges two blocks Params: base: dict { name: value } other: dict { name: value } common: dict { name: value } base_renames: Renames object other_renames: Renames object Returns: Dict CellName => Resource with merged contents ''' if not base_renames: base_renames = {} if not other_renames: other_renames = {} base_changes = compare(common, base) other_changes = compare(common, other) base_changes.renames = base_renames other_changes.renames = other_renames ret = copy(common) self._handle_deletion(base_changes, other_changes, ret) self._handle_deletion(other_changes, base_changes, ret) self._handle_creation(base_changes, other_changes, ret) self._handle_creation(other_changes, base_changes, ret) self._handle_modify(base_changes, other_changes, ret) self._handle_modify(other_changes, base_changes, ret) self._handle_renames(base_changes, other_changes, ret) self._handle_renames(other_changes, base_changes, ret) assert not base_changes.deleted, base_changes.deleted assert not base_changes.created, base_changes.created assert not base_changes.modified, base_changes.modified assert not base_changes.renames, base_changes.renames assert not other_changes.deleted, other_changes.deleted assert not other_changes.created, other_changes.created assert not other_changes.modified, other_changes.modified assert not other_changes.renames, other_changes.renames return ret
def test_delete_resource(self): edition_resources = {} changes = compare(self.last_version_resources, edition_resources) self.assertEqual(1, len(changes.deleted)) self.assertEqual(0, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(0, len(changes.modified)) self.assertEqual((self.r1, self.c1), changes.deleted['r1.h'])
def test_delete_resource(self): edition_resources = {} changes = compare(self.last_version_resources, edition_resources) self.assertEqual(1, len(changes.deleted)) self.assertEqual(0, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(0, len(changes.modified)) self.assertEqual((self.r1, self.c1), changes.deleted['r1.h'])
def test_diff_text(self): base = {'f1': 'Hola que tal\nBien Gracias\n', 'f2': 'Hasta luego\n'} other = {'f1': 'Hola que tal\nBien, Bien\n', 'f3': 'Hasta luego lucas\n'} changes = compare(base, other) diff = compute_diff(changes, text_unified_diff) self.assertIn('-Hasta luego', diff.deleted['f2']) self.assertIn('+Hasta luego lucas', diff.created['f3']) self.assertIn('-Bien Gracias', diff.modified['f1']) self.assertIn('+Bien, Bien', diff.modified['f1'])
def test_diff_text(self): base = {'f1': 'Hola que tal\nBien Gracias\n', 'f2': 'Hasta luego\n'} other = { 'f1': 'Hola que tal\nBien, Bien\n', 'f3': 'Hasta luego lucas\n' } changes = compare(base, other) diff = compute_diff(changes, text_unified_diff) self.assertIn('-Hasta luego', diff.deleted['f2']) self.assertIn('+Hasta luego lucas', diff.created['f3']) self.assertIn('-Bien Gracias', diff.modified['f1']) self.assertIn('+Bien, Bien', diff.modified['f1'])
def test_modify_content(self): '''just modify the content of a file''' r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('bye')) edition_resources = {'r1.h': Resource(r1, content)} changes = compare(self.last_version_resources, edition_resources) self.assertEqual(0, len(changes.deleted)) self.assertEqual(0, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(1, len(changes.modified)) self.assertEqual(Resource(r1, content), changes.modified['r1.h'].new)
def test_modify_content(self): '''just modify the content of a file''' r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('bye')) edition_resources = {'r1.h': Resource(r1, content)} changes = compare(self.last_version_resources, edition_resources) self.assertEqual(0, len(changes.deleted)) self.assertEqual(0, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(1, len(changes.modified)) self.assertEqual(Resource(r1, content), changes.modified['r1.h'].new)
def test_create_resource(self): r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('hello')) r2 = SimpleCell(self.brl_block.block_name + 'r2.h') content2 = Content(id_=None, load=Blob('bye')) edition_resources = {'r1.h': Resource(r1, content), 'r2.h': Resource(r2, content2)} changes = compare(self.last_version_resources, edition_resources) self.assertEqual(0, len(changes.deleted)) self.assertEqual(1, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(0, len(changes.modified)) self.assertEqual(Resource(r2, content2), changes.created['r2.h'])
def test_modify_content_diff(self): r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('bye')) edition_resources = {'r1.h': Resource(r1, content)} changes = compare(self.last_version_resources, edition_resources) diff = compute_diff(changes, resource_diff_function) self.assertEqual(0, len(diff.deleted)) self.assertEqual(0, len(diff.created)) self.assertEqual(0, len(diff.renames)) self.assertEqual(1, len(diff.modified)) self.assertEqual('--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye', diff.modified['r1.h'].content) #print '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye' self.assertEqual(Resource(None, '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'), diff.modified['r1.h'])
def test_deduce_renames_multi_different_values(self): '''2 is deleted 3 is created with 3 4 is created with 4 2 is considered to be renamed to 3 ''' base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)} other_resources = {CellName('1'): Int(1), CellName('3'): Int(3), CellName('4'): Int(4)} #compute changes without renames changes = compare(base_resources, other_resources) changes.deduce_renames() #nothing changes self.assertEqual({CellName('2'): 2}, changes.deleted) self.assertEqual({CellName('3'): Int(3), CellName('4'): 4}, changes.created) self.assertEqual({}, changes.modified) self.assertEqual({CellName('2'): CellName('3')}, changes.renames)
def test_create_resource(self): r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('hello')) r2 = SimpleCell(self.brl_block.block_name + 'r2.h') content2 = Content(id_=None, load=Blob('bye')) edition_resources = { 'r1.h': Resource(r1, content), 'r2.h': Resource(r2, content2) } changes = compare(self.last_version_resources, edition_resources) self.assertEqual(0, len(changes.deleted)) self.assertEqual(1, len(changes.created)) self.assertEqual(0, len(changes.renames)) self.assertEqual(0, len(changes.modified)) self.assertEqual(Resource(r2, content2), changes.created['r2.h'])
def test_modify_content_diff(self): r1 = SimpleCell(self.brl_block.block_name + 'r1.h') content = Content(id_=None, load=Blob('bye')) edition_resources = {'r1.h': Resource(r1, content)} changes = compare(self.last_version_resources, edition_resources) diff = compute_diff(changes, resource_diff_function) self.assertEqual(0, len(diff.deleted)) self.assertEqual(0, len(diff.created)) self.assertEqual(0, len(diff.renames)) self.assertEqual(1, len(diff.modified)) self.assertEqual( '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye', diff.modified['r1.h'].content) #print '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye' self.assertEqual( Resource(None, '--- base\n\n+++ other\n\n@@ -1 +1 @@\n\n-hello\n+bye'), diff.modified['r1.h'])
def test_deduce_renames_multi_all_equal(self): '''2 is deleted 3 is created with 2's contents 4 is created with 2's contents 2 is considered to be renamed to 4 ''' #FIXME: Conclusion is arbitrary. Last one to be processed with equal similarty degreee # is the one choosen. We might have to inform the user about this base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)} other_resources = {CellName('1'): Int(1), CellName('3'): Int(2), CellName('4'): Int(2)} #compute changes without renames changes = compare(base_resources, other_resources) changes.deduce_renames() #nothing changes self.assertEqual({CellName('2'): 2}, changes.deleted) self.assertEqual({CellName('3'): Int(2), CellName('4'): 2}, changes.created) self.assertEqual({}, changes.modified) self.assertEqual({CellName('2'): CellName('4')}, changes.renames)
def build_publish_request(biiapi, hive_holder, block_name, tag, msg, versiontag, origin, biiout): block_name, dep_block_names = _check_input(hive_holder, block_name) _check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout) block_holder = hive_holder[block_name] if not changevalidator.check_block_size(block_holder, biiout): raise PublishException("Block is too large to be published") parent = block_holder.parent _check_possible(parent, biiapi, biiout) if parent.time != -1: # Update remote_block_holder = biiapi.get_block_holder(parent) base_resources = remote_block_holder.resources parent_delta_info = biiapi.get_version_delta_info(parent) else: # New block base_resources = None parent_delta_info = None remote_block_holder = None changes = compare(base_resources, block_holder.resources) if not block_changed(changes, block_holder, remote_block_holder): if parent_delta_info and tag > parent_delta_info.tag: biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag)) changes.modified.pop(BIICODE_FILE, None) else: raise UpToDatePublishException("Up to date, nothing to publish") changes.deduce_renames() request = PublishRequest() request.parent = parent request.changes = changes if parent_delta_info: request.parent_time = parent_delta_info.date assert all(bv.time is not None for bv in block_holder.requirements.itervalues()) request.deptable = block_holder.requirements request.tag = tag request.msg = msg request.versiontag = versiontag request.origin = origin return request
def test_deduce_renames(self): ''' 2 is modified from 2 to 22 3 is deleted 5 is renamed to 6 (5 deleted, 6 created) 10 is created''' base_resources = { CellName('1'): Int(1), CellName('2'): Int(2), CellName('3'): Int(3), CellName('4'): Int(4), CellName('5'): Int(5) } other_resources = { CellName('1'): Int(1), CellName('2'): Int(22), CellName('4'): Int(4), CellName('6'): Int(6), CellName('10'): Int(10) } #compute changes without renames changes = compare(base_resources, other_resources) self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted) self.assertEqual({ CellName('6'): Int(6), CellName('10'): 10 }, changes.created) self.assertEqual({CellName('2'): (2, 22)}, changes.modified) self.assertEqual({}, changes.renames) #deduce renames changes.deduce_renames() #nothing changes self.assertEqual({CellName('3'): 3, CellName('5'): 5}, changes.deleted) self.assertEqual({ CellName('6'): Int(6), CellName('10'): 10 }, changes.created) self.assertEqual({CellName('2'): (2, 22)}, changes.modified) #but the renames field self.assertEqual({CellName('5'): CellName('6')}, changes.renames)
def build_publish_request(biiapi, hive_holder, block_name, tag, msg, versiontag, origin, biiout): block_name, dep_block_names = _check_input(hive_holder, block_name) _check_dep_blocks(biiapi, hive_holder, dep_block_names, biiout) block_holder = hive_holder[block_name] if not changevalidator.check_block_size(block_holder, biiout): raise PublishException("Block is too large to be published") parent = block_holder.parent _check_possible(parent, biiapi, biiout) if parent.time != -1: # Update remote_block_holder = biiapi.get_block_holder(parent) base_resources = remote_block_holder.resources parent_delta_info = biiapi.get_version_delta_info(parent) else: # New block base_resources = None parent_delta_info = None remote_block_holder = None changes = compare(base_resources, block_holder.resources) if not block_changed(changes, block_holder, remote_block_holder): if parent_delta_info and tag > parent_delta_info.tag: biiout.info('No changes, promoting tag %s -> %s' % (parent_delta_info.tag, tag)) changes.modified.pop(BIICODE_FILE, None) else: raise UpToDatePublishException("Up to date, nothing to publish") changes.deduce_renames() request = PublishRequest() request.parent = parent request.changes = changes if parent_delta_info: request.parent_time = parent_delta_info.date assert all(bv.time is not None for bv in block_holder.requirements.itervalues()) request.deptable = block_holder.requirements request.tag = tag request.msg = msg request.versiontag = versiontag request.origin = origin return request
def test_deduce_renames_multi_different_values(self): '''2 is deleted 3 is created with 3 4 is created with 4 2 is considered to be renamed to 3 ''' base_resources = {CellName('1'): Int(1), CellName('2'): Int(2)} other_resources = { CellName('1'): Int(1), CellName('3'): Int(3), CellName('4'): Int(4) } #compute changes without renames changes = compare(base_resources, other_resources) changes.deduce_renames() #nothing changes self.assertEqual({CellName('2'): 2}, changes.deleted) self.assertEqual({ CellName('3'): Int(3), CellName('4'): 4 }, changes.created) self.assertEqual({}, changes.modified) self.assertEqual({CellName('2'): CellName('3')}, changes.renames)
def update(block_holder, time, biiapi, biiout): block_name = block_holder.block_name parent = block_holder.parent if parent is None: raise BiiException('No parent info in block: %s' % block_name) biiout.info("Updating %s" % parent.to_pretty()) if time is None: block_info = biiapi.get_block_info(parent.block) block_version = block_info.last_version time = block_version.time if time < parent.time: raise BiiException( 'Impossible to update block %s to an older version.\n' 'Current version %d > Requested version %d' % (parent.block.to_pretty(), parent.time, time)) # Tags for text merge base_tag_name = 'current' other_tag_name = 'v%d' % time # Prepare merge info base_resources = block_holder.resources other_version = BlockVersion(parent.block, time) common_holder = biiapi.get_block_holder(parent) common_holder.parent = parent common_holder.commit_config() common_resources = common_holder.resources if time == parent.time: # For updating with self parent other_holder = common_holder common_resources = base_resources # The common is the base, otherwise outdate other_renames = {} else: other_holder = biiapi.get_block_holder(other_version) other_holder.parent = parent other_holder.commit_config() other_renames = biiapi.get_renames(parent.block, parent.time, time) other_resources = other_holder.resources content_changes = compare(common_resources, base_resources) content_changes.deduce_renames() base_renames = content_changes.renames # Merge resources base_blobs = { name: content.load for name, (_, content) in base_resources.iteritems() if content is not None } other_blobs = { name: content.load for name, (_, content) in other_resources.iteritems() if content is not None } common_blobs = { name: content.load for name, (_, content) in common_resources.iteritems() if content is not None } merger = BlobsMerger(base_tag_name, other_tag_name, biiout) result = merger.merge(base_blobs, other_blobs, common_blobs, base_renames, other_renames) final_result = {name: blob.load for name, blob in result.iteritems()} return final_result, other_version