def test_default_fs_conn_id(self): with tempfile.NamedTemporaryFile() as tmp: task = FileSensor( task_id="test", filepath=tmp.name[1:], dag=self.dag, timeout=0, ) task._hook = self.hook task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
def test_wildcard_file(self): suffix = '.txt' with tempfile.NamedTemporaryFile(suffix=suffix) as tmp: fileglob = os.path.join(os.path.dirname(tmp.name), '*' + suffix) task = FileSensor( task_id='test', filepath=fileglob, fs_conn_id='fs_default', dag=self.dag, timeout=0, ) task._hook = self.hook task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
def test_empty_dir(self): temp_dir = tempfile.mkdtemp() task = FileSensor(task_id="test", filepath=temp_dir[1:], fs_conn_id='fs_default', dag=self.dag, timeout=0, poke_interval=1) task._hook = self.hook try: with self.assertRaises(AirflowSensorTimeout): task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) finally: shutil.rmtree(temp_dir)
def test_subdirectory_empty(self): temp_dir = tempfile.mkdtemp() tempfile.mkdtemp(dir=temp_dir) task = FileSensor(task_id='test', filepath=temp_dir, fs_conn_id='fs_default', dag=self.dag, timeout=0, poke_interval=1) task._hook = self.hook with self.assertRaises(AirflowSensorTimeout): task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) shutil.rmtree(temp_dir)
def test_file_in_dir(self): temp_dir = tempfile.mkdtemp() task = FileSensor( task_id="test", filepath=temp_dir[1:], fs_conn_id='fs_default', dag=self.dag, timeout=0, ) task._hook = self.hook try: # `touch` the dir open(temp_dir + "/file", "a").close() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) finally: shutil.rmtree(temp_dir)
def test_subdirectory_not_empty(self): suffix = '.txt' temp_dir = tempfile.mkdtemp() subdir = tempfile.mkdtemp(dir=temp_dir) with tempfile.NamedTemporaryFile(suffix=suffix, dir=subdir): task = FileSensor( task_id='test', filepath=temp_dir, fs_conn_id='fs_default', dag=self.dag, timeout=0, ) task._hook = self.hook task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) shutil.rmtree(temp_dir)