Beispiel #1
0
 def test_compare_key_equal(self):
     """
     Confirms checking compare key works.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     src_file = FileStat(src='',
                         dest='',
                         compare_key='comparator_test.py',
                         size=10,
                         last_update=time,
                         src_type='local',
                         dest_type='s3',
                         operation_name='upload')
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     src_files.append(src_file)
     dest_files.append(dest_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #2
0
 def test_compare_key_greater(self):
     """
     Confirm the appropriate action is taken when the soruce compare key
     is greater than the destination compare key.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     src_file = FileStat(src='', dest='',
                         compare_key='domparator_test.py', size=10,
                         last_update=time, src_type='local',
                         dest_type='s3', operation_name='upload')
     dest_file = FileStat(src='', dest='',
                          compare_key='comparator_test.py', size=10,
                          last_update=time, src_type='s3',
                          dest_type='local', operation_name='')
     src_files.append(src_file)
     dest_files.append(dest_file)
     src_file.operation = 'upload'
     dest_file.operation = 'delete'
     ref_list.append(dest_file)
     ref_list.append(src_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #3
0
 def test_compare_key_less(self):
     """
     Confirm the appropriate action is taken when the soruce compare key
     is less than the destination compare key.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     src_file = FileStat(src='',
                         dest='',
                         compare_key='bomparator_test.py',
                         size=10,
                         last_update=time,
                         src_type='local',
                         dest_type='s3',
                         operation_name='upload')
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     src_files.append(src_file)
     dest_files.append(dest_file)
     dest_file.operation = 'delete'
     ref_list.append(src_file)
     ref_list.append(dest_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #4
0
    def test_compare_exact_timestamps_src_older(self):
        """
        Confirm that same-sized files are synced when
        the source is older than the destination and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
        time_dst = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)
Beispiel #5
0
    def test_compare_size_only_src_older_than_dest(self):
        """
        Confirm that files with the same size but different update times are not
        synced when `size_only` is set.
        """
        time_dst = datetime.datetime.now()
        time_src = time_dst + datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='s3',
                            dest_type='local',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)
Beispiel #6
0
    def test_compare_exact_timestamps_same_age_diff_size(self):
        """
        Confirm that files of differing sizes are synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=20,
                            last_update=time_both,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)
Beispiel #7
0
 def test_compare_lastmod_copy(self):
     """
     Confirms compare time works for copies.
     """
     time = datetime.datetime.now()
     future_time = time + datetime.timedelta(0, 3)
     src_file = FileStat(src='',
                         dest='',
                         compare_key='comparator_test.py',
                         size=10,
                         last_update=future_time,
                         src_type='s3',
                         dest_type='s3',
                         operation_name='copy')
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='s3',
                          operation_name='')
     should_sync = self.sync_strategy.determine_should_sync(
         src_file, dest_file)
     self.assertTrue(should_sync)
Beispiel #8
0
 def test_compare_lastmod_upload(self):
     """
     Confirms compare time works for uploads.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     future_time = time + datetime.timedelta(0, 3)
     src_file = FileStat(src='',
                         dest='',
                         compare_key='comparator_test.py',
                         size=10,
                         last_update=future_time,
                         src_type='local',
                         dest_type='s3',
                         operation_name='upload')
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     src_files.append(src_file)
     dest_files.append(dest_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     ref_list.append(src_file)
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #9
0
    def test_compare_size_only_different_update_times(self):
        """
        Confirm that files with the same size but different update times
        are not synced.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src + datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='s3',
                            dest_type='local',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dst_file)
        self.assertFalse(should_sync)
Beispiel #10
0
 def test_local_directory(self):
     """
     Generate an entire local directory.
     """
     input_local_dir = {'src': {'path': self.local_dir,
                                'type': 'local'},
                        'dest': {'path': 'bucket/',
                                 'type': 's3'},
                        'dir_op': True, 'use_src_name': True}
     params = {'region': 'us-east-1'}
     files = FileGenerator(self.client, '').call(input_local_dir)
     result_list = []
     for filename in files:
         result_list.append(filename)
     size, last_update = get_file_stat(self.local_file)
     file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt',
                          compare_key='text1.txt', size=size,
                          last_update=last_update, src_type='local',
                          dest_type='s3', operation_name='')
     path = self.local_dir + 'another_directory' + os.sep \
         + 'text2.txt'
     size, last_update = get_file_stat(path)
     file_stat2 = FileStat(src=path,
                           dest='bucket/another_directory/text2.txt',
                           compare_key='another_directory/text2.txt',
                           size=size, last_update=last_update,
                           src_type='local',
                           dest_type='s3', operation_name='')
     ref_list = [file_stat2, file_stat]
     self.assertEqual(len(result_list), len(ref_list))
     for i in range(len(result_list)):
         compare_files(self, result_list[i], ref_list[i])
    def test_s3_directory(self):
        #
        # Generates s3 files under a common prefix. Also it ensures that
        # zero size files are ignored.
        # Note: Size and last update are not tested because s3 generates them.
        #
        input_s3_file = {'src': {'path': self.bucket+'/', 'type': 's3'},
                         'dest': {'path': '', 'type': 'local'},
                         'dir_op': True, 'use_src_name': True}
        result_list = list(
            FileGenerator(self.service, self.endpoint, '').call(
                input_s3_file))
        file_stat = FileStat(src=self.file2,
                             dest='another_directory' + os.sep + 'text2.txt',
                             compare_key='another_directory/text2.txt',
                             size=21,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local', operation_name='')
        file_stat2 = FileStat(src=self.file1,
                              dest='text1.txt',
                              compare_key='text1.txt',
                              size=15,
                              last_update=result_list[1].last_update,
                              src_type='s3',
                              dest_type='local', operation_name='')

        expected_result = [file_stat, file_stat2]
        self.assertEqual(len(result_list), 2)
        compare_files(self, result_list[0], expected_result[0])
        compare_files(self, result_list[1], expected_result[1])
Beispiel #12
0
    def test_compare_exact_timestamps_same_age_same_size(self):
        """
        Confirm that same-sized files are not synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dst_file)
        self.assertFalse(should_sync)
Beispiel #13
0
    def test_compare_exact_timestamps_src_older(self):
        """
        Confirm that same-sized files are synced when
        the source is older than the destination and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
        time_dst = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dst_file)
        self.assertTrue(should_sync)
Beispiel #14
0
    def test_compare_exact_timestamps_diff_age_not_download(self):
        """
        Confirm that same sized files are synced when the timestamps differ,
        the type of operation is not a download, and ``exact_timestamps``
        is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src - datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='upload')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dst_file)
        self.assertTrue(should_sync)
    def test_s3_directory(self):
        """
        Generates s3 files under a common prefix. Also it ensures that
        zero size files are ignored.
        Note: Size and last update are not tested because s3 generates them.
        """
        input_s3_file = {'src': {'path': self.bucket + '/', 'type': 's3'},
                         'dest': {'path': '', 'type': 'local'},
                         'dir_op': True, 'use_src_name': True}
        params = {'region': 'us-east-1'}
        files = FileGenerator(self.service, self.endpoint,
                              '').call(input_s3_file)
        result_list = []
        for filename in files:
            result_list.append(filename)
        file_stat = FileStat(src=self.file2,
                             dest='another_directory' + os.sep +
                             'text2.txt',
                             compare_key='another_directory/text2.txt',
                             size=result_list[0].size,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local', operation_name='')
        file_stat2 = FileStat(src=self.file1,
                              dest='text1.txt',
                              compare_key='text1.txt',
                              size=result_list[1].size,
                              last_update=result_list[1].last_update,
                              src_type='s3',
                              dest_type='local', operation_name='')

        ref_list = [file_stat, file_stat2]
        self.assertEqual(len(result_list), len(ref_list))
        for i in range(len(result_list)):
            compare_files(self, result_list[i], ref_list[i])
Beispiel #16
0
 def test_empty_src(self):
     """
     Confirm the appropriate action is taken when there are no more source
     files to take.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     dest_files.append(dest_file)
     dest_file.operation = 'delete'
     ref_list.append(dest_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #17
0
    def test_s3_directory(self):
        """
        Generates s3 files under a common prefix. Also it ensures that
        zero size files are ignored.
        Note: Size and last update are not tested because s3 generates them.
        """
        input_s3_file = {
            'src': {
                'path': self.bucket + '/',
                'type': 's3'
            },
            'dest': {
                'path': '',
                'type': 'local'
            },
            'dir_op': True,
            'use_src_name': True
        }
        params = {'region': 'us-east-1'}
        files = FileGenerator(self.client, '').call(input_s3_file)

        self.parsed_responses = [{
            "CommonPrefixes": [],
            "Contents": [{
                "Key": "another_directory/text2.txt",
                "Size": 100,
                "LastModified": "2014-01-09T20:45:49.000Z"
            }, {
                "Key": "text1.txt",
                "Size": 10,
                "LastModified": "2013-01-09T20:45:49.000Z"
            }]
        }]
        self.patch_make_request()
        result_list = []
        for filename in files:
            result_list.append(filename)
        file_stat = FileStat(src=self.file2,
                             dest='another_directory' + os.sep + 'text2.txt',
                             compare_key='another_directory/text2.txt',
                             size=result_list[0].size,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')
        file_stat2 = FileStat(src=self.file1,
                              dest='text1.txt',
                              compare_key='text1.txt',
                              size=result_list[1].size,
                              last_update=result_list[1].last_update,
                              src_type='s3',
                              dest_type='local',
                              operation_name='')

        ref_list = [file_stat, file_stat2]
        self.assertEqual(len(result_list), len(ref_list))
        for i in range(len(result_list)):
            compare_files(self, result_list[i], ref_list[i])
    def test_s3_delete_directory(self):
        #
        # Generates s3 files under a common prefix. Also it ensures that
        # the directory itself is included because it is a delete command
        # Note: Size and last update are not tested because s3 generates them.
        #
        input_s3_file = {
            'src': {
                'path': self.bucket + '/',
                'type': 's3'
            },
            'dest': {
                'path': '',
                'type': 'local'
            },
            'dir_op': True,
            'use_src_name': True
        }
        result_list = list(
            FileGenerator(self.client, 'delete').call(input_s3_file))

        file_stat1 = FileStat(src=self.bucket + '/another_directory/',
                              dest='another_directory' + os.sep,
                              compare_key='another_directory/',
                              size=0,
                              last_update=result_list[0].last_update,
                              src_type='s3',
                              dest_type='local',
                              operation_name='delete')
        file_stat2 = FileStat(src=self.file2,
                              dest='another_directory' + os.sep + 'text2.txt',
                              compare_key='another_directory/text2.txt',
                              size=21,
                              last_update=result_list[1].last_update,
                              src_type='s3',
                              dest_type='local',
                              operation_name='delete')
        file_stat3 = FileStat(src=self.file1,
                              dest='text1.txt',
                              compare_key='text1.txt',
                              size=15,
                              last_update=result_list[2].last_update,
                              src_type='s3',
                              dest_type='local',
                              operation_name='delete')

        expected_list = [file_stat1, file_stat2, file_stat3]
        self.assertEqual(len(result_list), 3)
        compare_files(self, result_list[0], expected_list[0])
        compare_files(self, result_list[1], expected_list[1])
        compare_files(self, result_list[2], expected_list[2])
Beispiel #19
0
    def test_compare_lastmod_download(self):
        """
        Confirms compare time works for downloads.
        """
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileStat(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=future_time,
                             src_type='local',
                             dest_type='s3',
                             operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dest_file)
        self.assertTrue(should_sync)

        # If the source is newer than the destination do not download.
        src_file = FileStat(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=future_time,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='local',
                             dest_type='s3',
                             operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(
            src_file, dest_file)
        self.assertFalse(should_sync)
    def test_s3_file(self):
        """
        Generate a single s3 file
        Note: Size and last update are not tested because s3 generates them.
        """
        input_s3_file = {'src': {'path': self.file1, 'type': 's3'},
                         'dest': {'path': 'text1.txt', 'type': 'local'},
                         'dir_op': False, 'use_src_name': False}
        params = {'region': 'us-east-1'}
        file_gen = FileGenerator(self.service, self.endpoint, '')
        files = file_gen.call(input_s3_file)
        result_list = []
        for filename in files:
            result_list.append(filename)
        file_stat = FileStat(src=self.file1, dest='text1.txt',
                             compare_key='text1.txt',
                             size=result_list[0].size,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local', operation_name='')

        ref_list = [file_stat]
        self.assertEqual(len(result_list), len(ref_list))
        for i in range(len(result_list)):
            compare_files(self, result_list[i], ref_list[i])
Beispiel #21
0
 def test_s3_single_file_delete(self):
     input_s3_file = {
         'src': {
             'path': self.file1,
             'type': 's3'
         },
         'dest': {
             'path': '',
             'type': 'local'
         },
         'dir_op': False,
         'use_src_name': True
     }
     self.client = mock.Mock()
     file_gen = FileGenerator(self.client, 'delete')
     result_list = list(file_gen.call(input_s3_file))
     self.assertEqual(len(result_list), 1)
     compare_files(
         self, result_list[0],
         FileStat(src=self.file1,
                  dest='text1.txt',
                  compare_key='text1.txt',
                  size=None,
                  last_update=None,
                  src_type='s3',
                  dest_type='local',
                  operation_name='delete'))
     self.client.head_object.assert_not_called()
Beispiel #22
0
 def test_local_file(self):
     """
     Generate a single local file.
     """
     input_local_file = {
         'src': {
             'path': self.local_file,
             'type': 'local'
         },
         'dest': {
             'path': 'bucket/text1.txt',
             'type': 's3'
         },
         'dir_op': False,
         'use_src_name': False
     }
     params = {'region': 'us-east-1'}
     files = FileGenerator(self.client, '').call(input_local_file)
     result_list = []
     for filename in files:
         result_list.append(filename)
     size, last_update = get_file_stat(self.local_file)
     file_stat = FileStat(src=self.local_file,
                          dest='bucket/text1.txt',
                          compare_key='text1.txt',
                          size=size,
                          last_update=last_update,
                          src_type='local',
                          dest_type='s3',
                          operation_name='')
     ref_list = [file_stat]
     self.assertEqual(len(result_list), len(ref_list))
     for i in range(len(result_list)):
         compare_files(self, result_list[i], ref_list[i])
    def test_s3_file(self):
        #
        # Generate a single s3 file
        # Note: Size and last update are not tested because s3 generates them.
        #
        input_s3_file = {
            'src': {
                'path': self.file1,
                'type': 's3'
            },
            'dest': {
                'path': 'text1.txt',
                'type': 'local'
            },
            'dir_op': False,
            'use_src_name': False
        }
        expected_file_size = 15
        result_list = list(
            FileGenerator(self.service, self.endpoint, '').call(input_s3_file))
        file_stat = FileStat(src=self.file1,
                             dest='text1.txt',
                             compare_key='text1.txt',
                             size=expected_file_size,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')

        expected_list = [file_stat]
        self.assertEqual(len(result_list), 1)
        compare_files(self, result_list[0], expected_list[0])
 def test_page_size(self):
     input_s3_file = {
         'src': {
             'path': self.bucket + '/',
             'type': 's3'
         },
         'dest': {
             'path': '',
             'type': 'local'
         },
         'dir_op': True,
         'use_src_name': True
     }
     file_gen = FileGenerator(self.service, self.endpoint, '',
                              page_size=1).call(input_s3_file)
     limited_file_gen = itertools.islice(file_gen, 1)
     result_list = list(limited_file_gen)
     file_stat = FileStat(src=self.file2,
                          dest='another_directory' + os.sep + 'text2.txt',
                          compare_key='another_directory/text2.txt',
                          size=21,
                          last_update=result_list[0].last_update,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     # Ensure only one item is returned from ``ListObjects``
     self.assertEqual(len(result_list), 1)
     compare_files(self, result_list[0], file_stat)
Beispiel #25
0
    def test_s3_file(self):
        """
        Generate a single s3 file
        Note: Size and last update are not tested because s3 generates them.
        """
        input_s3_file = {'src': {'path': self.file1, 'type': 's3'},
                         'dest': {'path': 'text1.txt', 'type': 'local'},
                         'dir_op': False, 'use_src_name': False}
        params = {'region': 'us-east-1'}
        self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100,
                                  "LastModified": "2014-01-09T20:45:49.000Z"}]
        self.patch_make_request()

        file_gen = FileGenerator(self.client, '')
        files = file_gen.call(input_s3_file)
        result_list = []
        for filename in files:
            result_list.append(filename)
        file_stat = FileStat(src=self.file1, dest='text1.txt',
                             compare_key='text1.txt',
                             size=result_list[0].size,
                             last_update=result_list[0].last_update,
                             src_type='s3',
                             dest_type='local', operation_name='')

        ref_list = [file_stat]
        self.assertEqual(len(result_list), len(ref_list))
        for i in range(len(result_list)):
            compare_files(self, result_list[i], ref_list[i])
Beispiel #26
0
    def test_empty_dest(self):
        """
        Confirm the appropriate action is taken when there are no more dest
        files to take.
        """
        # Try when the sync strategy says to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='', dest='',
                            compare_key='domparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload')
        src_files.append(src_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)
Beispiel #27
0
 def test_empty_src(self):
     """
     Confirm the appropriate action is taken when there are no more source
     files to take.
     """
     src_files = []
     dest_files = []
     ref_list = []
     result_list = []
     time = datetime.datetime.now()
     dest_file = FileStat(src='', dest='',
                          compare_key='comparator_test.py', size=10,
                          last_update=time, src_type='s3',
                          dest_type='local', operation_name='')
     dest_files.append(dest_file)
     dest_file.operation = 'delete'
     ref_list.append(dest_file)
     files = self.comparator.call(iter(src_files), iter(dest_files))
     for filename in files:
         result_list.append(filename)
     self.assertEqual(result_list, ref_list)
Beispiel #28
0
 def file_stat(self, filename, src_type='local'):
     if src_type == 'local':
         filename = os.path.abspath(filename)
         dest_type = 's3'
     else:
         dest_type = 'local'
     return FileStat(src=filename,
                     dest='',
                     compare_key='',
                     size=10,
                     last_update=0,
                     src_type=src_type,
                     dest_type=dest_type,
                     operation_name='')
Beispiel #29
0
    def test_determine_should_sync(self):
        time_src = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(src_file, None)
        self.assertTrue(should_sync)
Beispiel #30
0
    def test_determine_should_sync(self):
        timenow = datetime.datetime.now()

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=timenow,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        should_sync = self.sync_strategy.determine_should_sync(None, dst_file)
        self.assertTrue(should_sync)
        self.assertEqual(dst_file.operation_name, 'delete')
Beispiel #31
0
 def test_compare_size(self):
     """
     Confirms compare size works.
     """
     time = datetime.datetime.now()
     src_file = FileStat(src='',
                         dest='',
                         compare_key='comparator_test.py',
                         size=11,
                         last_update=time,
                         src_type='local',
                         dest_type='s3',
                         operation_name='upload')
     dest_file = FileStat(src='',
                          dest='',
                          compare_key='comparator_test.py',
                          size=10,
                          last_update=time,
                          src_type='s3',
                          dest_type='local',
                          operation_name='')
     should_sync = self.sync_strategy.determine_should_sync(
         src_file, dest_file)
     self.assertTrue(should_sync)
 def test_info_setter(self):
     info_setter = FileInfoBuilder(client='client',
                                   source_client='source_client',
                                   parameters='parameters',
                                   is_stream='is_stream')
     files = [
         FileStat(src='src',
                  dest='dest',
                  compare_key='compare_key',
                  size='size',
                  last_update='last_update',
                  src_type='src_type',
                  dest_type='dest_type',
                  operation_name='operation_name')
     ]
     file_infos = info_setter.call(files)
     for file_info in file_infos:
         attributes = file_info.__dict__.keys()
         for key in attributes:
             self.assertEqual(getattr(file_info, key), str(key))