def test_compare_key_equal(self): """ Confirms checking compare key works. """ src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def test_compare_key_greater(self): """ Confirm the appropriate action is taken when the soruce compare key is greater than the destination compare key. """ src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) src_file.operation = 'upload' dest_file.operation = 'delete' ref_list.append(dest_file) ref_list.append(src_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def test_compare_key_less(self): """ Confirm the appropriate action is taken when the soruce compare key is less than the destination compare key. """ src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='bomparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) dest_file.operation = 'delete' ref_list.append(src_file) ref_list.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def test_compare_exact_timestamps_src_older(self): """ Confirm that same-sized files are synced when the source is older than the destination and `exact_timestamps` is set. """ time_src = datetime.datetime.now() - datetime.timedelta(days=1) time_dst = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='download') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 1)
def test_compare_size_only_src_older_than_dest(self): """ Confirm that files with the same size but different update times are not synced when `size_only` is set. """ time_dst = datetime.datetime.now() time_src = time_dst + datetime.timedelta(days=1) src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', dest_type='s3', operation_name='upload') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', dest_type='local', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 0)
def test_compare_exact_timestamps_same_age_diff_size(self): """ Confirm that files of differing sizes are synced when the source and destination are the same age and `exact_timestamps` is set. """ time_both = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='test.py', size=20, last_update=time_both, src_type='s3', dest_type='local', operation_name='download') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 1)
def test_compare_lastmod_copy(self): """ Confirms compare time works for copies. """ time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', dest_type='s3', operation_name='copy') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dest_file) self.assertTrue(should_sync)
def test_compare_lastmod_upload(self): """ Confirms compare time works for uploads. """ src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) ref_list.append(src_file) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def test_compare_size_only_different_update_times(self): """ Confirm that files with the same size but different update times are not synced. """ time_src = datetime.datetime.now() time_dst = time_src + datetime.timedelta(days=1) src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', dest_type='s3', operation_name='upload') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', dest_type='local', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dst_file) self.assertFalse(should_sync)
def test_local_directory(self): """ Generate an entire local directory. """ input_local_dir = {'src': {'path': self.local_dir, 'type': 'local'}, 'dest': {'path': 'bucket/', 'type': 's3'}, 'dir_op': True, 'use_src_name': True} params = {'region': 'us-east-1'} files = FileGenerator(self.client, '').call(input_local_dir) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') path = self.local_dir + 'another_directory' + os.sep \ + 'text2.txt' size, last_update = get_file_stat(path) file_stat2 = FileStat(src=path, dest='bucket/another_directory/text2.txt', compare_key='another_directory/text2.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') ref_list = [file_stat2, file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_s3_directory(self): # # Generates s3 files under a common prefix. Also it ensures that # zero size files are ignored. # Note: Size and last update are not tested because s3 generates them. # input_s3_file = {'src': {'path': self.bucket+'/', 'type': 's3'}, 'dest': {'path': '', 'type': 'local'}, 'dir_op': True, 'use_src_name': True} result_list = list( FileGenerator(self.service, self.endpoint, '').call( input_s3_file)) file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') file_stat2 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=15, last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='') expected_result = [file_stat, file_stat2] self.assertEqual(len(result_list), 2) compare_files(self, result_list[0], expected_result[0]) compare_files(self, result_list[1], expected_result[1])
def test_compare_exact_timestamps_same_age_same_size(self): """ Confirm that same-sized files are not synced when the source and destination are the same age and `exact_timestamps` is set. """ time_both = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='s3', dest_type='local', operation_name='download') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dst_file) self.assertFalse(should_sync)
def test_compare_exact_timestamps_src_older(self): """ Confirm that same-sized files are synced when the source is older than the destination and `exact_timestamps` is set. """ time_src = datetime.datetime.now() - datetime.timedelta(days=1) time_dst = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='download') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dst_file) self.assertTrue(should_sync)
def test_compare_exact_timestamps_diff_age_not_download(self): """ Confirm that same sized files are synced when the timestamps differ, the type of operation is not a download, and ``exact_timestamps`` is set. """ time_src = datetime.datetime.now() time_dst = time_src - datetime.timedelta(days=1) src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='upload') dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dst_file) self.assertTrue(should_sync)
def test_s3_directory(self): """ Generates s3 files under a common prefix. Also it ensures that zero size files are ignored. Note: Size and last update are not tested because s3 generates them. """ input_s3_file = {'src': {'path': self.bucket + '/', 'type': 's3'}, 'dest': {'path': '', 'type': 'local'}, 'dir_op': True, 'use_src_name': True} params = {'region': 'us-east-1'} files = FileGenerator(self.service, self.endpoint, '').call(input_s3_file) result_list = [] for filename in files: result_list.append(filename) file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') file_stat2 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[1].size, last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='') ref_list = [file_stat, file_stat2] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_empty_src(self): """ Confirm the appropriate action is taken when there are no more source files to take. """ src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') dest_files.append(dest_file) dest_file.operation = 'delete' ref_list.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def test_s3_directory(self): """ Generates s3 files under a common prefix. Also it ensures that zero size files are ignored. Note: Size and last update are not tested because s3 generates them. """ input_s3_file = { 'src': { 'path': self.bucket + '/', 'type': 's3' }, 'dest': { 'path': '', 'type': 'local' }, 'dir_op': True, 'use_src_name': True } params = {'region': 'us-east-1'} files = FileGenerator(self.client, '').call(input_s3_file) self.parsed_responses = [{ "CommonPrefixes": [], "Contents": [{ "Key": "another_directory/text2.txt", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z" }, { "Key": "text1.txt", "Size": 10, "LastModified": "2013-01-09T20:45:49.000Z" }] }] self.patch_make_request() result_list = [] for filename in files: result_list.append(filename) file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') file_stat2 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[1].size, last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='') ref_list = [file_stat, file_stat2] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_s3_delete_directory(self): # # Generates s3 files under a common prefix. Also it ensures that # the directory itself is included because it is a delete command # Note: Size and last update are not tested because s3 generates them. # input_s3_file = { 'src': { 'path': self.bucket + '/', 'type': 's3' }, 'dest': { 'path': '', 'type': 'local' }, 'dir_op': True, 'use_src_name': True } result_list = list( FileGenerator(self.client, 'delete').call(input_s3_file)) file_stat1 = FileStat(src=self.bucket + '/another_directory/', dest='another_directory' + os.sep, compare_key='another_directory/', size=0, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='delete') file_stat2 = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='delete') file_stat3 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=15, last_update=result_list[2].last_update, src_type='s3', dest_type='local', operation_name='delete') expected_list = [file_stat1, file_stat2, file_stat3] self.assertEqual(len(result_list), 3) compare_files(self, result_list[0], expected_list[0]) compare_files(self, result_list[1], expected_list[1]) compare_files(self, result_list[2], expected_list[2])
def test_compare_lastmod_download(self): """ Confirms compare time works for downloads. """ time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='download') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dest_file) self.assertTrue(should_sync) # If the source is newer than the destination do not download. src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', dest_type='local', operation_name='download') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dest_file) self.assertFalse(should_sync)
def test_s3_file(self): """ Generate a single s3 file Note: Size and last update are not tested because s3 generates them. """ input_s3_file = {'src': {'path': self.file1, 'type': 's3'}, 'dest': {'path': 'text1.txt', 'type': 'local'}, 'dir_op': False, 'use_src_name': False} params = {'region': 'us-east-1'} file_gen = FileGenerator(self.service, self.endpoint, '') files = file_gen.call(input_s3_file) result_list = [] for filename in files: result_list.append(filename) file_stat = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_s3_single_file_delete(self): input_s3_file = { 'src': { 'path': self.file1, 'type': 's3' }, 'dest': { 'path': '', 'type': 'local' }, 'dir_op': False, 'use_src_name': True } self.client = mock.Mock() file_gen = FileGenerator(self.client, 'delete') result_list = list(file_gen.call(input_s3_file)) self.assertEqual(len(result_list), 1) compare_files( self, result_list[0], FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=None, last_update=None, src_type='s3', dest_type='local', operation_name='delete')) self.client.head_object.assert_not_called()
def test_local_file(self): """ Generate a single local file. """ input_local_file = { 'src': { 'path': self.local_file, 'type': 'local' }, 'dest': { 'path': 'bucket/text1.txt', 'type': 's3' }, 'dir_op': False, 'use_src_name': False } params = {'region': 'us-east-1'} files = FileGenerator(self.client, '').call(input_local_file) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_s3_file(self): # # Generate a single s3 file # Note: Size and last update are not tested because s3 generates them. # input_s3_file = { 'src': { 'path': self.file1, 'type': 's3' }, 'dest': { 'path': 'text1.txt', 'type': 'local' }, 'dir_op': False, 'use_src_name': False } expected_file_size = 15 result_list = list( FileGenerator(self.service, self.endpoint, '').call(input_s3_file)) file_stat = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=expected_file_size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') expected_list = [file_stat] self.assertEqual(len(result_list), 1) compare_files(self, result_list[0], expected_list[0])
def test_page_size(self): input_s3_file = { 'src': { 'path': self.bucket + '/', 'type': 's3' }, 'dest': { 'path': '', 'type': 'local' }, 'dir_op': True, 'use_src_name': True } file_gen = FileGenerator(self.service, self.endpoint, '', page_size=1).call(input_s3_file) limited_file_gen = itertools.islice(file_gen, 1) result_list = list(limited_file_gen) file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') # Ensure only one item is returned from ``ListObjects`` self.assertEqual(len(result_list), 1) compare_files(self, result_list[0], file_stat)
def test_s3_file(self): """ Generate a single s3 file Note: Size and last update are not tested because s3 generates them. """ input_s3_file = {'src': {'path': self.file1, 'type': 's3'}, 'dest': {'path': 'text1.txt', 'type': 'local'}, 'dir_op': False, 'use_src_name': False} params = {'region': 'us-east-1'} self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, "LastModified": "2014-01-09T20:45:49.000Z"}] self.patch_make_request() file_gen = FileGenerator(self.client, '') files = file_gen.call(input_s3_file) result_list = [] for filename in files: result_list.append(filename) file_stat = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_empty_dest(self): """ Confirm the appropriate action is taken when there are no more dest files to take. """ # Try when the sync strategy says to sync the file. self.not_at_dest_sync_strategy.determine_should_sync.return_value = True src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') src_files.append(src_file) ref_list.append(src_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) # Now try when the sync strategy says not to sync the file. self.not_at_dest_sync_strategy.determine_should_sync.return_value = False result_list = [] ref_list = [] files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list)
def file_stat(self, filename, src_type='local'): if src_type == 'local': filename = os.path.abspath(filename) dest_type = 's3' else: dest_type = 'local' return FileStat(src=filename, dest='', compare_key='', size=10, last_update=0, src_type=src_type, dest_type=dest_type, operation_name='')
def test_determine_should_sync(self): time_src = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='') should_sync = self.sync_strategy.determine_should_sync(src_file, None) self.assertTrue(should_sync)
def test_determine_should_sync(self): timenow = datetime.datetime.now() dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=timenow, src_type='local', dest_type='s3', operation_name='') should_sync = self.sync_strategy.determine_should_sync(None, dst_file) self.assertTrue(should_sync) self.assertEqual(dst_file.operation_name, 'delete')
def test_compare_size(self): """ Confirms compare size works. """ time = datetime.datetime.now() src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=11, last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') should_sync = self.sync_strategy.determine_should_sync( src_file, dest_file) self.assertTrue(should_sync)
def test_info_setter(self): info_setter = FileInfoBuilder(client='client', source_client='source_client', parameters='parameters', is_stream='is_stream') files = [ FileStat(src='src', dest='dest', compare_key='compare_key', size='size', last_update='last_update', src_type='src_type', dest_type='dest_type', operation_name='operation_name') ] file_infos = info_setter.call(files) for file_info in file_infos: attributes = file_info.__dict__.keys() for key in attributes: self.assertEqual(getattr(file_info, key), str(key))