Пример #1
0
 def setUp(self):
     self.sync_strategy = Mock()
     self.not_at_src_sync_strategy = Mock()
     self.not_at_dest_sync_strategy = Mock()
     self.comparator = Comparator(self.sync_strategy,
                                  self.not_at_dest_sync_strategy,
                                  self.not_at_src_sync_strategy)
Пример #2
0
class ComparatorSizeOnlyTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'delete': True, 'size_only': True})

    def test_compare_size_only_dest_older_than_src(self):
        """
        Confirm that files with the same size but different update times are not
        synced when `size_only` is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src + datetime.timedelta(days=1)

        src_file = FileInfo(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_src, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)

        dst_file = FileInfo(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_dst, src_type='s3',
                            dest_type='local', operation_name='',
                            service=None, endpoint=None)

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)

    def test_compare_size_only_src_older_than_dest(self):
        """
        Confirm that files with the same size but different update times are not
        synced when `size_only` is set.
        """
        time_dst = datetime.datetime.now()
        time_src = time_dst + datetime.timedelta(days=1)

        src_file = FileInfo(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_src, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)

        dst_file = FileInfo(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_dst, src_type='s3',
                            dest_type='local', operation_name='',
                            service=None, endpoint=None)

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)
Пример #3
0
 def setUp(self):
     self.sync_strategy = Mock()
     self.not_at_src_sync_strategy = Mock()
     self.not_at_dest_sync_strategy = Mock()
     self.comparator = Comparator(self.sync_strategy,
                                  self.not_at_dest_sync_strategy,
                                  self.not_at_src_sync_strategy)
Пример #4
0
    def run(self):
        """
        This function wires together all of the generators and completes
        the command.  First a dictionary is created that is indexed first by
        the command name.  Then using the instruction, another dictionary
        can be indexed to obtain the objects corresponding to the
        particular instruction for that command.  To begin the wiring,
        either a ``FileFormat`` or ``TaskInfo`` object, depending on the
        command, is put into a list.  Then the function enters a while loop
        that pops off an instruction.  It then determines the object needed
        and calls the call function of the object using the list as the input.
        Depending on the number of objects in the input list and the number
        of components in the list corresponding to the instruction, the call
        method of the component can be called two different ways.  If the
        number of inputs is equal to the number of components a 1:1 mapping of
        inputs to components is used when calling the call function.  If the
        there are more inputs than components, then a 2:1 mapping of inputs to
        components is used where the component call method takes two inputs
        instead of one.  Whatever files are yielded from the call function
        is appended to a list and used as the input for the next repetition
        of the while loop until there are no more instructions.
        """
        src = self.parameters['src']
        dest = self.parameters['dest']
        paths_type = self.parameters['paths_type']
        files = FileFormat().format(src, dest, self.parameters)
        rev_files = FileFormat().format(dest, src, self.parameters)

        cmd_translation = {}
        cmd_translation['locals3'] = {
            'cp': 'upload',
            'sync': 'upload',
            'mv': 'move'
        }
        cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
        cmd_translation['s3local'] = {
            'cp': 'download',
            'sync': 'download',
            'mv': 'move'
        }
        cmd_translation['s3'] = {
            'rm': 'delete',
            'ls': 'list_objects',
            'mb': 'make_bucket',
            'rb': 'remove_bucket'
        }
        operation = cmd_translation[paths_type][self.cmd]

        file_generator = FileGenerator(self.session, operation,
                                       self.parameters)
        rev_generator = FileGenerator(self.session, '', self.parameters)
        taskinfo = [
            TaskInfo(src=files['src']['path'],
                     src_type='s3',
                     operation=operation)
        ]
        s3handler = S3Handler(self.session, self.parameters)

        command_dict = {}
        command_dict['sync'] = {
            'setup': [files, rev_files],
            'file_generator': [file_generator, rev_generator],
            'filters': [Filter(self.parameters),
                        Filter(self.parameters)],
            'comparator': [Comparator(self.parameters)],
            's3_handler': [s3handler]
        }

        command_dict['cp'] = {
            'setup': [files],
            'file_generator': [file_generator],
            'filters': [Filter(self.parameters)],
            's3_handler': [s3handler]
        }

        command_dict['rm'] = {
            'setup': [files],
            'file_generator': [file_generator],
            'filters': [Filter(self.parameters)],
            's3_handler': [s3handler]
        }

        command_dict['mv'] = {
            'setup': [files],
            'file_generator': [file_generator],
            'filters': [Filter(self.parameters)],
            's3_handler': [s3handler]
        }

        command_dict['ls'] = {'setup': [taskinfo], 's3_handler': [s3handler]}

        command_dict['mb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}

        command_dict['rb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}

        files = command_dict[self.cmd]['setup']

        while self.instructions:
            instruction = self.instructions.pop(0)
            file_list = []
            components = command_dict[self.cmd][instruction]
            for i in range(len(components)):
                if len(files) > len(components):
                    file_list.append(components[i].call(*files))
                else:
                    file_list.append(components[i].call(files[i]))
            files = file_list
Пример #5
0
 def setUp(self):
     self.comparator = Comparator({"exact_timestamps": True})
Пример #6
0
class ComparatorExactTimestampsTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({"exact_timestamps": True})

    def test_compare_exact_timestamps_dest_older(self):
        """
        Confirm that same-sized files are synced when
        the destination is older than the source and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src - datetime.timedelta(days=1)

        src_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_src,
            src_type="s3",
            dest_type="local",
            operation_name="download",
            service=None,
            endpoint=None,
        )

        dst_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_dst,
            src_type="local",
            dest_type="s3",
            operation_name="",
            service=None,
            endpoint=None,
        )

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_src_older(self):
        """
        Confirm that same-sized files are synced when
        the source is older than the destination and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
        time_dst = datetime.datetime.now()

        src_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_src,
            src_type="s3",
            dest_type="local",
            operation_name="download",
            service=None,
            endpoint=None,
        )

        dst_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_dst,
            src_type="local",
            dest_type="s3",
            operation_name="",
            service=None,
            endpoint=None,
        )

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_same_age_same_size(self):
        """
        Confirm that same-sized files are not synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_both,
            src_type="s3",
            dest_type="local",
            operation_name="download",
            service=None,
            endpoint=None,
        )

        dst_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_both,
            src_type="local",
            dest_type="s3",
            operation_name="",
            service=None,
            endpoint=None,
        )

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)

    def test_compare_exact_timestamps_same_age_diff_size(self):
        """
        Confirm that files of differing sizes are synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=20,
            last_update=time_both,
            src_type="s3",
            dest_type="local",
            operation_name="download",
            service=None,
            endpoint=None,
        )

        dst_file = FileInfo(
            src="",
            dest="",
            compare_key="test.py",
            size=10,
            last_update=time_both,
            src_type="local",
            dest_type="s3",
            operation_name="",
            service=None,
            endpoint=None,
        )

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)
Пример #7
0
 def setUp(self):
     self.comparator = Comparator({"delete": True, "size_only": True})
Пример #8
0
class ComparatorTest(unittest.TestCase):
    def setUp(self):
        self.sync_strategy = Mock()
        self.not_at_src_sync_strategy = Mock()
        self.not_at_dest_sync_strategy = Mock()
        self.comparator = Comparator(self.sync_strategy,
                                     self.not_at_dest_sync_strategy,
                                     self.not_at_src_sync_strategy)

    def test_compare_key_equal_should_not_sync(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is equal to the destination compare key.
        """
        # Try when the sync strategy says not to sync the file.
        self.sync_strategy.determine_should_sync.return_value = False

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Try when the sync strategy says to sync the file.
        self.sync_strategy.determine_should_sync.return_value = True

        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_key_less(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is less than the destination compare key.
        """
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False

        # Try when the sync strategy says to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='',
                            dest='',
                            compare_key='bomparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_key_greater(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is greater than the destination compare key.
        """
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False

        # Try when the sync strategy says to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='',
                            dest='',
                            compare_key='domparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src(self):
        """
        Confirm the appropriate action is taken when there are no more source
        files to take.
        """
        # Try when the sync strategy says to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        dest_file = FileStat(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='')
        dest_files.append(dest_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_dest(self):
        """
        Confirm the appropriate action is taken when there are no more dest
        files to take.
        """
        # Try when the sync strategy says to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='',
                            dest='',
                            compare_key='domparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')
        src_files.append(src_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src_dest(self):
        """
        Confirm the appropriate action is taken when there are no more
        files to take for both source and destination.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)
Пример #9
0
    def run(self):
        """
        This function wires together all of the generators and completes
        the command.  First a dictionary is created that is indexed first by
        the command name.  Then using the instruction, another dictionary
        can be indexed to obtain the objects corresponding to the
        particular instruction for that command.  To begin the wiring,
        either a ``FileFormat`` or ``TaskInfo`` object, depending on the
        command, is put into a list.  Then the function enters a while loop
        that pops off an instruction.  It then determines the object needed
        and calls the call function of the object using the list as the input.
        Depending on the number of objects in the input list and the number
        of components in the list corresponding to the instruction, the call
        method of the component can be called two different ways.  If the
        number of inputs is equal to the number of components a 1:1 mapping of
        inputs to components is used when calling the call function.  If the
        there are more inputs than components, then a 2:1 mapping of inputs to
        components is used where the component call method takes two inputs
        instead of one.  Whatever files are yielded from the call function
        is appended to a list and used as the input for the next repetition
        of the while loop until there are no more instructions.
        """
        src = self.parameters['src']
        dest = self.parameters['dest']
        paths_type = self.parameters['paths_type']
        files = FileFormat().format(src, dest, self.parameters)
        rev_files = FileFormat().format(dest, src, self.parameters)

        cmd_translation = {}
        cmd_translation['locals3'] = {
            'cp': 'upload',
            'sync': 'upload',
            'mv': 'move'
        }
        cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
        cmd_translation['s3local'] = {
            'cp': 'download',
            'sync': 'download',
            'mv': 'move'
        }
        cmd_translation['s3'] = {
            'rm': 'delete',
            'mb': 'make_bucket',
            'rb': 'remove_bucket'
        }
        result_queue = queue.Queue()
        operation_name = cmd_translation[paths_type][self.cmd]

        fgen_kwargs = {
            'client': self._source_client,
            'operation_name': operation_name,
            'follow_symlinks': self.parameters['follow_symlinks'],
            'page_size': self.parameters['page_size'],
            'result_queue': result_queue
        }
        rgen_kwargs = {
            'client': self._client,
            'operation_name': '',
            'follow_symlinks': self.parameters['follow_symlinks'],
            'page_size': self.parameters['page_size'],
            'result_queue': result_queue
        }

        fgen_request_parameters = {}
        fgen_head_object_params = {}
        fgen_request_parameters['HeadObject'] = fgen_head_object_params
        fgen_kwargs['request_parameters'] = fgen_request_parameters

        # SSE-C may be neaded for HeadObject for copies/downloads/deletes
        # If the operation is s3 to s3, the FileGenerator should use the
        # copy source key and algorithm. Otherwise, use the regular
        # SSE-C key and algorithm. Note the reverse FileGenerator does
        # not need any of these because it is used only for sync operations
        # which only use ListObjects which does not require HeadObject.
        RequestParamsMapper.map_head_object_params(fgen_head_object_params,
                                                   self.parameters)
        if paths_type == 's3s3':
            RequestParamsMapper.map_head_object_params(
                fgen_head_object_params, {
                    'sse_c': self.parameters.get('sse_c_copy_source'),
                    'sse_c_key': self.parameters.get('sse_c_copy_source_key')
                })

        file_generator = FileGenerator(**fgen_kwargs)
        rev_generator = FileGenerator(**rgen_kwargs)
        taskinfo = [
            TaskInfo(src=files['src']['path'],
                     src_type='s3',
                     operation_name=operation_name,
                     client=self._client)
        ]
        stream_dest_path, stream_compare_key = find_dest_path_comp_key(files)
        stream_file_info = [
            FileInfo(src=files['src']['path'],
                     dest=stream_dest_path,
                     compare_key=stream_compare_key,
                     src_type=files['src']['type'],
                     dest_type=files['dest']['type'],
                     operation_name=operation_name,
                     client=self._client,
                     is_stream=True)
        ]
        file_info_builder = FileInfoBuilder(self._client, self._source_client,
                                            self.parameters)
        s3handler = S3Handler(self.session,
                              self.parameters,
                              runtime_config=self._runtime_config,
                              result_queue=result_queue)
        s3_stream_handler = S3StreamHandler(self.session,
                                            self.parameters,
                                            result_queue=result_queue)

        sync_strategies = self.choose_sync_strategies()

        command_dict = {}
        if self.cmd == 'sync':
            command_dict = {
                'setup': [files, rev_files],
                'file_generator': [file_generator, rev_generator],
                'filters': [
                    create_filter(self.parameters),
                    create_filter(self.parameters)
                ],
                'comparator': [Comparator(**sync_strategies)],
                'file_info_builder': [file_info_builder],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'cp' and self.parameters['is_stream']:
            command_dict = {
                'setup': [stream_file_info],
                's3_handler': [s3_stream_handler]
            }
        elif self.cmd == 'cp':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                'file_info_builder': [file_info_builder],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'rm':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                'file_info_builder': [file_info_builder],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'mv':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                'file_info_builder': [file_info_builder],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'mb':
            command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
        elif self.cmd == 'rb':
            command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}

        files = command_dict['setup']
        while self.instructions:
            instruction = self.instructions.pop(0)
            file_list = []
            components = command_dict[instruction]
            for i in range(len(components)):
                if len(files) > len(components):
                    file_list.append(components[i].call(*files))
                else:
                    file_list.append(components[i].call(files[i]))
            files = file_list
        # This is kinda quirky, but each call through the instructions
        # will replaces the files attr with the return value of the
        # file_list.  The very last call is a single list of
        # [s3_handler], and the s3_handler returns the number of
        # tasks failed and the number of tasks warned.
        # This means that files[0] now contains a namedtuple with
        # the number of failed tasks and the number of warned tasks.
        # In terms of the RC, we're keeping it simple and saying
        # that > 0 failed tasks will give a 1 RC and > 0 warned
        # tasks will give a 2 RC.  Otherwise a RC of zero is returned.
        rc = 0
        if files[0].num_tasks_failed > 0:
            rc = 1
        if files[0].num_tasks_warned > 0:
            rc = 2
        return rc
Пример #10
0
class ComparatorTest(unittest.TestCase):
    def setUp(self):
        self.sync_strategy = Mock()
        self.not_at_src_sync_strategy = Mock()
        self.not_at_dest_sync_strategy = Mock()
        self.comparator = Comparator(self.sync_strategy,
                                     self.not_at_dest_sync_strategy,
                                     self.not_at_src_sync_strategy)

    def test_compare_key_equal_should_not_sync(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is equal to the destination compare key.
        """
        # Try when the sync strategy says not to sync the file.
        self.sync_strategy.determine_should_sync.return_value = False

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload')
        dest_file = FileStat(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Try when the sync strategy says to sync the file.
        self.sync_strategy.determine_should_sync.return_value = True

        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_key_less(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is less than the destination compare key.
        """
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False

        # Try when the sync strategy says to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='', dest='',
                            compare_key='bomparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload')
        dest_file = FileStat(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)


    def test_compare_key_greater(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is greater than the destination compare key.
        """
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False

        # Try when the sync strategy says to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = True
        
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='', dest='',
                            compare_key='domparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload')
        dest_file = FileStat(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='')
        src_files.append(src_file)
        dest_files.append(dest_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)


    def test_empty_src(self):
        """
        Confirm the appropriate action is taken when there are no more source
        files to take.
        """
        # Try when the sync strategy says to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        dest_file = FileStat(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='')
        dest_files.append(dest_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_src_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_dest(self):
        """
        Confirm the appropriate action is taken when there are no more dest
        files to take.
        """
        # Try when the sync strategy says to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = True

        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileStat(src='', dest='',
                            compare_key='domparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload')
        src_files.append(src_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # Now try when the sync strategy says not to sync the file.
        self.not_at_dest_sync_strategy.determine_should_sync.return_value = False
        result_list = []
        ref_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)


    def test_empty_src_dest(self):
        """
        Confirm the appropriate action is taken when there are no more
        files to take for both source and destination.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)
Пример #11
0
 def setUp(self):
     self.comparator = Comparator({'exact_timestamps': True})
Пример #12
0
class ComparatorExactTimestampsTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'exact_timestamps': True})

    def test_compare_exact_timestamps_dest_older(self):
        """
        Confirm that same-sized files are synced when
        the destination is older than the source and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src - datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_src_older(self):
        """
        Confirm that same-sized files are synced when
        the source is older than the destination and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
        time_dst = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_same_age_same_size(self):
        """
        Confirm that same-sized files are not synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)

    def test_compare_exact_timestamps_same_age_diff_size(self):
        """
        Confirm that files of differing sizes are synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=20,
                            last_update=time_both,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_both,
                            src_type='local',
                            dest_type='s3',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)
Пример #13
0
 def setUp(self):
     self.comparator = Comparator({'delete': True, 'size_only': True})
Пример #14
0
class ComparatorSizeOnlyTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'delete': True, 'size_only': True})

    def test_compare_size_only_dest_older_than_src(self):
        """
        Confirm that files with the same size but different update times are not
        synced when `size_only` is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src + datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='s3',
                            dest_type='local',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)

    def test_compare_size_only_src_older_than_dest(self):
        """
        Confirm that files with the same size but different update times are not
        synced when `size_only` is set.
        """
        time_dst = datetime.datetime.now()
        time_src = time_dst + datetime.timedelta(days=1)

        src_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_src,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload')

        dst_file = FileStat(src='',
                            dest='',
                            compare_key='test.py',
                            size=10,
                            last_update=time_dst,
                            src_type='s3',
                            dest_type='local',
                            operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)
Пример #15
0
 def setUp(self):
     self.comparator = Comparator({'delete': True, 'size_only': True})
Пример #16
0
class ComparatorTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'delete': True})

    def test_compare_key_equal(self):
        """
        Confirms checking compare key works.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_size(self):
        """
        Confirms compare size works.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=11,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_upload(self):
        """
        Confirms compare time works for uploads.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=future_time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_copy(self):
        """
        Confirms compare time works for copies
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=future_time, src_type='s3',
                            dest_type='s3', operation_name='copy',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='s3', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_download(self):
        """
        Confirms compare time works for downloads.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=time, src_type='s3',
                            dest_type='local', operation_name='download',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=future_time, src_type='local',
                             dest_type='s3', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # If the source is newer than the destination do not download.
        src_file = FileInfo(src='', dest='',
                            compare_key='comparator_test.py', size=10,
                            last_update=future_time, src_type='s3',
                            dest_type='local', operation_name='download',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='local',
                             dest_type='s3', operation_name='',
                             service=None, endpoint=None)
        src_files = []
        dest_files = []
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        result_list = []
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, [])

    def test_compare_key_less(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is less than the destination compare key.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='', dest='',
                            compare_key='bomparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        dest_file.operation = 'delete'
        ref_list.append(src_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_key_greater(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is greater than the destination compare key.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='', dest='',
                            compare_key='domparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        src_file.operation = 'upload'
        dest_file.operation = 'delete'
        ref_list.append(dest_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src(self):
        """
        Confirm the appropriate action is taken when there are no more source
        files to take.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        dest_file = FileInfo(src='', dest='',
                             compare_key='comparator_test.py', size=10,
                             last_update=time, src_type='s3',
                             dest_type='local', operation_name='',
                             service=None, endpoint=None)
        dest_files.append(dest_file)
        dest_file.operation = 'delete'
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_dest(self):
        """
        Confirm the appropriate action is taken when there are no more dest
        files to take.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='', dest='',
                            compare_key='domparator_test.py', size=10,
                            last_update=time, src_type='local',
                            dest_type='s3', operation_name='upload',
                            service=None, endpoint=None)
        src_files.append(src_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src_dest(self):
        """
        Confirm the appropriate action is taken when there are no more
        files to take for both source and destination.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)
Пример #17
0
 def setUp(self):
     self.comparator = Comparator({'delete': True})
Пример #18
0
class ComparatorTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'delete': True})

    def test_compare_key_equal(self):
        """
        Confirms checking compare key works.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_size(self):
        """
        Confirms compare size works.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=11,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_upload(self):
        """
        Confirms compare time works for uploads.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=future_time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_copy(self):
        """
        Confirms compare time works for copies
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=future_time,
                            src_type='s3',
                            dest_type='s3',
                            operation_name='copy',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='s3',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_lastmod_download(self):
        """
        Confirms compare time works for downloads.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        future_time = time + datetime.timedelta(0, 3)
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=future_time,
                             src_type='local',
                             dest_type='s3',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        ref_list.append(src_file)
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

        # If the source is newer than the destination do not download.
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='comparator_test.py',
                            size=10,
                            last_update=future_time,
                            src_type='s3',
                            dest_type='local',
                            operation_name='download',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='local',
                             dest_type='s3',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files = []
        dest_files = []
        src_files.append(src_file)
        dest_files.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        result_list = []
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, [])

    def test_compare_key_less(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is less than the destination compare key.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='bomparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        dest_file.operation = 'delete'
        ref_list.append(src_file)
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_compare_key_greater(self):
        """
        Confirm the appropriate action is taken when the soruce compare key
        is greater than the destination compare key.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='domparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        src_files.append(src_file)
        dest_files.append(dest_file)
        src_file.operation = 'upload'
        dest_file.operation = 'delete'
        ref_list.append(dest_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src(self):
        """
        Confirm the appropriate action is taken when there are no more source
        files to take.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        dest_file = FileInfo(src='',
                             dest='',
                             compare_key='comparator_test.py',
                             size=10,
                             last_update=time,
                             src_type='s3',
                             dest_type='local',
                             operation_name='',
                             service=None,
                             endpoint=None)
        dest_files.append(dest_file)
        dest_file.operation = 'delete'
        ref_list.append(dest_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_dest(self):
        """
        Confirm the appropriate action is taken when there are no more dest
        files to take.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        time = datetime.datetime.now()
        src_file = FileInfo(src='',
                            dest='',
                            compare_key='domparator_test.py',
                            size=10,
                            last_update=time,
                            src_type='local',
                            dest_type='s3',
                            operation_name='upload',
                            service=None,
                            endpoint=None)
        src_files.append(src_file)
        ref_list.append(src_file)
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)

    def test_empty_src_dest(self):
        """
        Confirm the appropriate action is taken when there are no more
        files to take for both source and destination.
        """
        src_files = []
        dest_files = []
        ref_list = []
        result_list = []
        files = self.comparator.call(iter(src_files), iter(dest_files))
        for filename in files:
            result_list.append(filename)
        self.assertEqual(result_list, ref_list)
Пример #19
0
 def setUp(self):
     self.comparator = Comparator({'delete': True})
Пример #20
0
 def setUp(self):
     self.comparator = Comparator({"delete": True})
Пример #21
0
    def run(self):
        """
        This function wires together all of the generators and completes
        the command.  First a dictionary is created that is indexed first by
        the command name.  Then using the instruction, another dictionary
        can be indexed to obtain the objects corresponding to the
        particular instruction for that command.  To begin the wiring,
        either a ``FileFormat`` or ``TaskInfo`` object, depending on the
        command, is put into a list.  Then the function enters a while loop
        that pops off an instruction.  It then determines the object needed
        and calls the call function of the object using the list as the input.
        Depending on the number of objects in the input list and the number
        of components in the list corresponding to the instruction, the call
        method of the component can be called two different ways.  If the
        number of inputs is equal to the number of components a 1:1 mapping of
        inputs to components is used when calling the call function.  If the
        there are more inputs than components, then a 2:1 mapping of inputs to
        components is used where the component call method takes two inputs
        instead of one.  Whatever files are yielded from the call function
        is appended to a list and used as the input for the next repetition
        of the while loop until there are no more instructions.
        """
        src = self.parameters['src']
        dest = self.parameters['dest']
        paths_type = self.parameters['paths_type']
        files = FileFormat().format(src, dest, self.parameters)
        rev_files = FileFormat().format(dest, src, self.parameters)

        cmd_translation = {}
        cmd_translation['locals3'] = {
            'cp': 'upload',
            'sync': 'upload',
            'mv': 'move'
        }
        cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
        cmd_translation['s3local'] = {
            'cp': 'download',
            'sync': 'download',
            'mv': 'move'
        }
        cmd_translation['s3'] = {
            'rm': 'delete',
            'mb': 'make_bucket',
            'rb': 'remove_bucket'
        }
        operation_name = cmd_translation[paths_type][self.cmd]
        file_generator = FileGenerator(self._service, self._endpoint,
                                       operation_name,
                                       self.parameters['follow_symlinks'])
        rev_generator = FileGenerator(self._service, self._endpoint, '',
                                      self.parameters['follow_symlinks'])
        taskinfo = [
            TaskInfo(src=files['src']['path'],
                     src_type='s3',
                     operation_name=operation_name,
                     service=self._service,
                     endpoint=self._endpoint)
        ]
        s3handler = S3Handler(self.session, self.parameters)

        command_dict = {}
        if self.cmd == 'sync':
            command_dict = {
                'setup': [files, rev_files],
                'file_generator': [file_generator, rev_generator],
                'filters': [
                    create_filter(self.parameters),
                    create_filter(self.parameters)
                ],
                'comparator': [Comparator(self.parameters)],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'cp':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'rm':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'mv':
            command_dict = {
                'setup': [files],
                'file_generator': [file_generator],
                'filters': [create_filter(self.parameters)],
                's3_handler': [s3handler]
            }
        elif self.cmd == 'mb':
            command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
        elif self.cmd == 'rb':
            command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}

        files = command_dict['setup']
        while self.instructions:
            instruction = self.instructions.pop(0)
            file_list = []
            components = command_dict[instruction]
            for i in range(len(components)):
                if len(files) > len(components):
                    file_list.append(components[i].call(*files))
                else:
                    file_list.append(components[i].call(files[i]))
            files = file_list
        # This is kinda quirky, but each call through the instructions
        # will replaces the files attr with the return value of the
        # file_list.  The very last call is a single list of
        # [s3_handler], and the s3_handler returns the number of
        # tasks failed.  This means that files[0] now contains
        # the number of failed tasks.  In terms of the RC, we're
        # keeping it simple and saying that > 0 failed tasks
        # will give a 1 RC.
        rc = 0
        if files[0] > 0:
            rc = 1
        return rc
Пример #22
0
class ComparatorExactTimestampsTest(unittest.TestCase):
    def setUp(self):
        self.comparator = Comparator({'exact_timestamps': True})

    def test_compare_exact_timestamps_dest_older(self):
        """
        Confirm that same-sized files are synced when
        the destination is older than the source and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now()
        time_dst = time_src - datetime.timedelta(days=1)

        src_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_src, src_type='s3',
                            dest_type='local', operation_name='download')

        dst_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_dst, src_type='local',
                            dest_type='s3', operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_src_older(self):
        """
        Confirm that same-sized files are synced when
        the source is older than the destination and
        `exact_timestamps` is set.
        """
        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
        time_dst = datetime.datetime.now()

        src_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_src, src_type='s3',
                            dest_type='local', operation_name='download')

        dst_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_dst, src_type='local',
                            dest_type='s3', operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)

    def test_compare_exact_timestamps_same_age_same_size(self):
        """
        Confirm that same-sized files are not synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_both, src_type='s3',
                            dest_type='local', operation_name='download')

        dst_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_both, src_type='local',
                            dest_type='s3', operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 0)

    def test_compare_exact_timestamps_same_age_diff_size(self):
        """
        Confirm that files of differing sizes are synced when
        the source and destination are the same age and
        `exact_timestamps` is set.
        """
        time_both = datetime.datetime.now()

        src_file = FileStat(src='', dest='',
                            compare_key='test.py', size=20,
                            last_update=time_both, src_type='s3',
                            dest_type='local', operation_name='download')

        dst_file = FileStat(src='', dest='',
                            compare_key='test.py', size=10,
                            last_update=time_both, src_type='local',
                            dest_type='s3', operation_name='')

        files = self.comparator.call(iter([src_file]), iter([dst_file]))
        self.assertEqual(sum(1 for _ in files), 1)