def test_change_entry_conflict_to_dir(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict f = open(os.path.join(_TEST_SRC, 'test7_become_dir'), 'w') # Modify file to become dir with different content, conflict f.write(_TEST_CONFLICT_CONTENT) f.close() # Delete file already, no conflict os.remove(os.path.join(_TEST_SRC, 'dir2_modified', 'test2_7_become_dir')) # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'test7_become_dir (conflict copy 1)'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT)])
def test_change_entry_conflict_dir_deleted_but_modified(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Change the dir to file, which was deleted, no conflict f = open(os.path.join(_TEST_SRC, 'dir3_deleted', 'dir3_1_deleted', 'test3_1_1_deleted.txt'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir3_deleted'), change_entry.CONTENT_STATUS_NEW), (('.', 'dir3_deleted', 'dir3_1_deleted'), change_entry.CONTENT_STATUS_NEW), (('.', 'dir3_deleted', 'dir3_1_deleted', 'test3_1_1_deleted.txt'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT)])
def test_change_entry_conflict_to_dir_but_modified(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Modify the files in subdirectory of the dir to become file, conflict f = open(os.path.join(_TEST_SRC, 'dir10_become_file', 'test10_1_deleted.txt'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir10_become_file (conflict copy 1)'), change_entry.CONTENT_STATUS_NEW, 'dir10\n'), (('.', 'dir10_become_file'), change_entry.CONTENT_STATUS_TO_DIR), (('.', 'dir10_become_file', 'test10_1_deleted.txt'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT)])
def test_change_entry_conflict_file_modified(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict f = open(os.path.join(_TEST_SRC, 'test2_modified.txt'), 'w') # Modify file with different content, conflict f.write(_TEST_CONFLICT_CONTENT) f.close() f = open(os.path.join(_TEST_SRC, 'dir2_modified', 'test2_2_modified.txt'), 'w') # Modify file but same content, no conflict f.write('2_2\n') f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'test2_modified.txt'), change_entry.CONTENT_STATUS_MODIFIED, _TEST_CONFLICT_CONTENT), (('.', 'test2_modified (conflict copy 1).txt'), change_entry.CONTENT_STATUS_NEW, '2m\n')])
def test_change_entry_conflict_dir_modified_but_deleted(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Delete dir, which was modified, no conflict shutil.rmtree(os.path.join(_TEST_SRC, 'dir2_modified')) # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, [(('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir2_modified'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir2_modified', 'test2_1_unchanged.txt'), change_entry.CONTENT_STATUS_DELETED), (('.', 'dir2_modified', 'dir2_1_unchanged'), change_entry.CONTENT_STATUS_DELETED), (('.', 'dir2_modified', 'dir2_1_unchanged', 'test2_1_1_unchanged.txt'), change_entry.CONTENT_STATUS_DELETED)])
def test_change_entry_applied_to_dir_info(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old) di_final = change_entry.apply_dir_changes_to_dir_info('.', dc) dc_final = change_entry.get_dir_changes(di_new, di_final) self._assertDirChanges(dc_final)
def test_change_entry_conflict_to_dir_already(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Change to dir already with adding new file and modify the file to be # added, conflict os.remove(os.path.join(_TEST_SRC, 'test7_become_dir')) os.mkdir(os.path.join(_TEST_SRC, 'test7_become_dir')) f = open(os.path.join(_TEST_SRC, 'test7_become_dir', 'test_conflict.txt'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() f = open(os.path.join(_TEST_SRC, 'test7_become_dir', 'testtest7_1_new.txt'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'test7_become_dir'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'test7_become_dir', 'test_conflict.txt'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT), (('.', 'test7_become_dir', 'testtest7_1_new (conflict copy 1).txt'), change_entry.CONTENT_STATUS_NEW, 'test7_1\n'), (('.', 'test7_become_dir', 'testtest7_1_new.txt'), change_entry.CONTENT_STATUS_MODIFIED, _TEST_CONFLICT_CONTENT)])
def test_change_entry_conflict_dir_new(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict f = open(os.path.join(_TEST_SRC, 'dir11_new'), 'w') # New file, not a directory, conflict f.write(_TEST_CONFLICT_CONTENT) f.close() # New dir, add a new file, no conflict os.mkdir(os.path.join(_TEST_SRC, 'dir2_modified', 'dir2_11_new')) f = open(os.path.join(_TEST_SRC, 'dir2_modified', 'dir2_11_new', 'test_conflict_new.txt'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir11_new (conflict copy 1)'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT), (('.', 'dir2_modified'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir2_modified', 'dir2_11_new'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'dir2_modified', 'dir2_11_new', 'test_conflict_new.txt'), change_entry.CONTENT_STATUS_NEW, _TEST_CONFLICT_CONTENT)])
def test_change_entry_conflict_to_file_but_deleted(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Delete the dir already, no conflict shutil.rmtree(os.path.join(_TEST_SRC, 'dir10_become_file')) # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges(dc_final)
def test_change_entry_conflict_to_file_already(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict # Change the dir to file already, but different content, conflict shutil.rmtree(os.path.join(_TEST_SRC, 'dir10_become_file')) f = open(os.path.join(_TEST_SRC, 'dir10_become_file'), 'w') f.write(_TEST_CONFLICT_CONTENT) f.close() # Change the dir to file already, same content, no conflict shutil.rmtree( os.path.join(_TEST_SRC, 'dir2_modified', 'dir2_10_become_file')) f = open(os.path.join(_TEST_SRC, 'dir2_modified', 'dir2_10_become_file'), 'w') f.write('dir2_10\n') f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), # Existing file has the content already there (('.', 'dir10_become_file'), change_entry.CONTENT_STATUS_MODIFIED, _TEST_CONFLICT_CONTENT), # Conflict copy has the content from the sync (('.', 'dir10_become_file (conflict copy 1)'), change_entry.CONTENT_STATUS_NEW, 'dir10\n')])
def test_csv_read_write(self): dir_info = file_info.load_dir_info(_TEST_CASES_SRC) output = cStringIO.StringIO() dir_info.write_to_csv(output) dir_info_from_csv = file_info.load_dir_info_from_csv( cStringIO.StringIO(output.getvalue()), _TEST_CASES_SRC) file_info_list = [x for x in dir_info.flat_file_info_list()] file_info_list_from_csv = [ x for x in dir_info_from_csv.flat_file_info_list()] self.assertEqual(len(file_info_list), len(file_info_list_from_csv)) for i in range(len(file_info_list)): self._assert_file_info_list_valid_and_equal( file_info_list[i], file_info_list_from_csv[i])
def test_load_dir_info(self): file_info_list = [x for x in file_info.load_dir_info(_TEST_CASES_SRC).flat_file_info_list()] expected_file_info_list_from_csv = [x for x in file_info .load_dir_info_from_csv( open('expected_src_file_info_list.csv', 'r'), _TEST_CASES_SRC) .flat_file_info_list()] self.assertEqual(len(expected_file_info_list_from_csv), len(file_info_list)) for i in range(len(file_info_list)): file_info_list[i].calculate_hash() self._assert_file_info_list_valid_and_equal( expected_file_info_list_from_csv[i], file_info_list[i])
def test_change_entry_conflict_file_new_multiple_copy(self): os.chdir(_TEST_DEST) di_new = file_info.load_dir_info('.', calculate_hash=True) os.chdir(_TEST_SRC) di_old = file_info.load_dir_info('.', calculate_hash=True) dc = change_entry.get_dir_changes(di_new, di_old, root_dir=_TEST_DEST, tmp_dir=_TEST_TMP) # Generate conflict f = open(os.path.join(_TEST_SRC, 'test8_new.txt'), 'w') # New file with different content, conflict f.write(_TEST_CONFLICT_CONTENT) f.close() # New file which takes the conflict copy 1 name f = open(os.path.join(_TEST_SRC, 'test8_new (conflict copy 1).txt'), 'w') f.write('conflict again') f.close() # Apply dir changes to _TEST_SRC change_entry.apply_dir_changes_to_dir(_TEST_SRC, dc) # Verify the dir status os.chdir(_TEST_SRC) di_final = file_info.load_dir_info('.', calculate_hash=True) dc_final = change_entry.get_dir_changes(di_final, di_new) self._assertDirChanges( dc_final, special_cases=[ (('.'), change_entry.CONTENT_STATUS_MODIFIED), (('.', 'test8_new.txt'), change_entry.CONTENT_STATUS_MODIFIED, _TEST_CONFLICT_CONTENT), (('.', 'test8_new (conflict copy 1).txt'), change_entry.CONTENT_STATUS_NEW, 'conflict again'), (('.', 'test8_new (conflict copy 2).txt'), change_entry.CONTENT_STATUS_NEW, '8\n')])
def sync(self, old_dir_info, debug=False, verbose=False): tstart = time.time() cwd = os.getcwd() working_old_di = old_dir_info cloud_old_di = self._extract_compressed_dir_info(old_dir_info) if debug: print '============== Latency till extract input dir info : %s s' % ( time.time() - tstart) if verbose: phase = 1 print 'Phase %s: Examine changes on working_dir at %s' % ( phase, self.working_dir) os.chdir(self.working_dir) working_cur_di = file_info.load_dir_info('.', calculate_hash=True) working_dc = change_entry.get_dir_changes(working_cur_di, working_old_di, root_dir=self.working_dir, tmp_dir=self.tmp_dir, verbose=verbose) is_working_no_change = self._is_no_change(working_dc) working_dc = self._generate_compressed_dir_changes(working_dc) if debug: print '============== Latency after examing working_dir: %s s' % ( time.time() - tstart) self._print_changes('********** working_dc', working_dc) if verbose: phase += 1 print 'Phase %s: Examine changes on wrap_dir at %s' % ( phase, self.cloud_dir) os.chdir(self.cloud_dir) cloud_cur_di = file_info.load_dir_info( '.', calculate_hash=True, key=self.compression_key) cloud_dc = change_entry.get_dir_changes(cloud_cur_di, cloud_old_di, root_dir=self.cloud_dir, tmp_dir=self.tmp_dir, verbose=verbose) is_cloud_no_change = self._is_no_change(cloud_dc) cloud_dc_result = self._generate_original_dir_changes(cloud_dc) cloud_dc = cloud_dc_result[0] invalid_archives_dc_working = cloud_dc_result[1] invalid_archives_dc_cloud = cloud_dc_result[2] if invalid_archives_dc_working: if debug: self._print_changes('************ invalid_archives_dc', invalid_archives_dc_working) # Apply invalid_archives_dc to working dir because they are not # compressed. These files will be compressed and sync after next sync. change_entry.apply_dir_changes_to_dir(self.working_dir, invalid_archives_dc_working, verbose=verbose) change_entry.apply_dir_changes_to_dir(self.cloud_dir, invalid_archives_dc_cloud, verbose=verbose) if debug: print '============== Latency after examing wrap_dir: %s s' % ( time.time() - tstart) has_changes = not is_working_no_change or not is_cloud_no_change if has_changes and verbose: phase += 1 print 'Phase %s: Merge changes on both dirs' % phase result = merge.merge(working_dc, cloud_dc) working_dc_new = result[0] working_dc_old = result[1] cloud_dc_new = self._extract_compressed_dir_changes(result[2]) cloud_dc_old = result[3] working_dc_conflict = result[4] cloud_dc_conflict = self._extract_compressed_dir_changes( working_dc_conflict) if debug: print '============== Latency after merge: %s s' % (time.time() - tstart) self._print_changes('********** working_dc_conflict', working_dc_conflict) self._print_changes('********** cloud_dc_conflict', cloud_dc_conflict) if has_changes and verbose: phase += 1 print 'Phase %s: Apply merged changes on working_dir at %s' % ( phase, self.working_dir) change_entry.apply_dir_changes_to_dir(self.working_dir, working_dc_new, verbose=verbose) change_entry.apply_dir_changes_to_dir( self.working_dir, working_dc_conflict, force_conflict=change_entry.CONFLICT_NEW, verbose=verbose) working_di = change_entry.apply_dir_changes_to_dir_info('.', working_dc_old) if debug: print '============== Latency after applying merged changes on working_dir: %s' % (time.time() - tstart) if has_changes and verbose: phase += 1 print 'Phase %s: Apply merged changes on wrap_dir at %s' % ( phase, self.cloud_dir) change_entry.apply_dir_changes_to_dir(self.cloud_dir, cloud_dc_new, verbose=verbose) change_entry.apply_dir_changes_to_dir( self.cloud_dir, cloud_dc_conflict, force_conflict=change_entry.CONFLICT_NEW, verbose=verbose) cloud_di = change_entry.apply_dir_changes_to_dir_info('.', cloud_dc_old) if debug: print '============== Latency after applying merged changes on wrap_dir: %s' % (time.time() - tstart) os.chdir(cwd) return [has_changes, working_di, cloud_di]