def test_should_match_regex(self):
"""
test the empty directory behaviour
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
task.execute(None)
def test_should_match_regex(self):
"""
test the empty directory behaviour
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
task.execute(None)
def test_should_not_match_regex(self):
"""
test the empty directory behaviour
:return:
"""
# Given
self.logger.debug('#' * 10)
self.logger.debug('Running %s', self._testMethodName)
self.logger.debug('#' * 10)
compiled_regex = re.compile("^IDoNotExist")
task = HdfsSensorRegex(task_id='Should_not_match_the_regex',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
# Then
with self.assertRaises(AirflowSensorTimeout):
task.execute(None)
def test_should_not_match_regex(self):
"""
test the empty directory behaviour
:return:
"""
# Given
self.logger.debug('#' * 10)
self.logger.debug('Running %s', self._testMethodName)
self.logger.debug('#' * 10)
compiled_regex = re.compile("^IDoNotExist")
task = HdfsSensorRegex(task_id='Should_not_match_the_regex',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
# Then
with self.assertRaises(AirflowSensorTimeout):
task.execute(None)
def test_should_match_regex_but_filesize(self):
"""
test the file size behaviour with regex
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex_but_filesize',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
file_size=20,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
# Then
with self.assertRaises(AirflowSensorTimeout):
task.execute(None)
def test_should_match_regex_and_filesize(self):
"""
test the file size behaviour with regex
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex_and_filesize',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
ignore_copying=True,
ignored_ext=['_COPYING_', 'sftp'],
file_size=10,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
task.execute(None)
def test_should_match_regex_but_filesize(self):
"""
test the file size behaviour with regex
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex_but_filesize',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
file_size=20,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
# Then
with self.assertRaises(AirflowSensorTimeout):
task.execute(None)
def test_should_match_regex_and_filesize(self):
"""
test the file size behaviour with regex
:return:
"""
# Given
self.log.debug('#' * 10)
self.log.debug('Running %s', self._testMethodName)
self.log.debug('#' * 10)
compiled_regex = re.compile("test[1-2]file")
task = HdfsSensorRegex(task_id='Should_match_the_regex_and_filesize',
filepath='/datadirectory/regex_dir',
regex=compiled_regex,
ignore_copying=True,
ignored_ext=['_COPYING_', 'sftp'],
file_size=10,
timeout=1,
retry_delay=timedelta(seconds=1),
poke_interval=1,
hook=self.hook)
# When
task.execute(None)