Exemplo n.º 1
0
    def test_should_match_regex(self):
        """
        test the empty directory behaviour
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        task.execute(None)
Exemplo n.º 2
0
    def test_should_match_regex(self):
        """
        test the empty directory behaviour
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        task.execute(None)
Exemplo n.º 3
0
    def test_should_match_regex_but_filesize(self):
        """
        test the file size behaviour with regex
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex_but_filesize',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               file_size=20,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        # Then
        with self.assertRaises(AirflowSensorTimeout):
            task.execute(None)
Exemplo n.º 4
0
    def test_should_match_regex_and_filesize(self):
        """
        test the file size behaviour with regex
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex_and_filesize',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               ignore_copying=True,
                               ignored_ext=['_COPYING_', 'sftp'],
                               file_size=10,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        task.execute(None)
Exemplo n.º 5
0
    def test_should_match_regex_but_filesize(self):
        """
        test the file size behaviour with regex
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex_but_filesize',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               file_size=20,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        # Then
        with self.assertRaises(AirflowSensorTimeout):
            task.execute(None)
Exemplo n.º 6
0
    def test_should_match_regex_and_filesize(self):
        """
        test the file size behaviour with regex
        :return:
        """
        # Given
        self.log.debug('#' * 10)
        self.log.debug('Running %s', self._testMethodName)
        self.log.debug('#' * 10)
        compiled_regex = re.compile("test[1-2]file")
        task = HdfsSensorRegex(task_id='Should_match_the_regex_and_filesize',
                               filepath='/datadirectory/regex_dir',
                               regex=compiled_regex,
                               ignore_copying=True,
                               ignored_ext=['_COPYING_', 'sftp'],
                               file_size=10,
                               timeout=1,
                               retry_delay=timedelta(seconds=1),
                               poke_interval=1,
                               hook=self.hook)

        # When
        task.execute(None)