def spark_session(request: FixtureRequest) -> SparkSession: # make sure env variables are set correctly if "SPARK_HOME" not in os.environ: os.environ["SPARK_HOME"] = "/usr/local/opt/spark" clean_spark_dir() master = "local[2]" session = ( SparkSession.builder.appName( f"pytest-pyspark-local-testing-{get_random_string(4)}" ) .master(master) .config("spark.ui.showConsoleProgress", "false") .config("spark.sql.shuffle.partitions", "2") .config("spark.default.parallelism", "4") .config("spark.driver.bindAddress", "127.0.0.1") .config("spark.sql.broadcastTimeout", "2400") .enableHiveSupport() .getOrCreate() ) request.addfinalizer(lambda: clean_close(session)) quiet_py4j() return session
def database(app: Flask, request: FixtureRequest) -> SQLAlchemy: if os.path.exists(TEST_DATABASE_PATH): os.unlink(TEST_DATABASE_PATH) db.app = app db.create_all() # Bypass pysqlite's broken transactions (see https://bit.ly/2DKiixa). # pylint:disable=unused-argument # pylint:disable=unused-variable @event.listens_for(db.engine, 'connect') def do_connect(connection: Connection, *args) -> None: connection.isolation_level = None # pylint:disable=unused-variable @event.listens_for(db.engine, 'begin') def do_begin(connection: Connection) -> None: connection.execute('BEGIN') def teardown() -> None: db.drop_all() os.unlink(TEST_DATABASE_PATH) request.addfinalizer(teardown) return db
def spark_session(request: FixtureRequest) -> SparkSession: # make sure env variables are set correctly if 'SPARK_HOME' not in os.environ: os.environ['SPARK_HOME'] = '/usr/local/opt/spark' clean_spark_dir() master = "local[2]" # master: str = "spark://localhost:7077" if not path.exists("/Applications/Docker.app"): master = "local[2]" print(f"++++++ Running on local spark: {master} ++++") else: print(f"++++++ Running on docker spark: {master} ++++") session = SparkSession.builder.appName(f"pytest-pyspark-local-testing-{get_random_string(4)}"). \ master(master). \ config("spark.ui.showConsoleProgress", "false"). \ config("spark.sql.shuffle.partitions", "2"). \ config("spark.default.parallelism", "4"). \ config("spark.sql.broadcastTimeout", "2400"). \ enableHiveSupport(). \ getOrCreate() request.addfinalizer(lambda: clean_close(session)) quiet_py4j() return session
def initialize_tests(request: FixtureRequest) -> None: initializer( ["eventual_tortoise.relation"], db_url="sqlite://:memory:", app_label="default", ) request.addfinalizer(finalizer)
def never_sleep(request: FixtureRequest) -> None: patcher = mock.patch("asyncio.sleep", _nosleep) patcher.start() def tearDown() -> None: patcher.stop() request.addfinalizer(tearDown)
def stop_std_logging(request: FixtureRequest, capfd: CaptureFixture) -> None: patcher = mock.patch("bandersnatch.log.setup_logging") patcher.start() def tearDown() -> None: patcher.stop() request.addfinalizer(tearDown)
def job_folder(request: FixtureRequest) -> None: """Create job folder and add finalizer to remove the folders after running the test.""" if not os.path.isdir(settings.JOB_FOLDER): os.makedirs(settings.JOB_FOLDER) def fin() -> None: shutil.rmtree(settings.JOB_FOLDER) request.addfinalizer(fin)
def selenium(request: FixtureRequest) -> webdriver.Remote: if request.config.getoption('--local-chrome'): driver = initialize_local_chrome() request.addfinalizer(lambda: deinitialize_selenium(driver)) return driver driver = request.getfixturevalue('selenium') # type: webdriver.Remote yield driver # https://github.com/SeleniumHQ/docker-selenium/issues/87#issuecomment-286231268 driver.close()
def logging_mock(request: FixtureRequest) -> mock.MagicMock: patcher = mock.patch("logging.config.fileConfig") logger: mock.MagicMock = patcher.start() def tearDown() -> None: patcher.stop() request.addfinalizer(tearDown) return logger
def mirror_mock(request: FixtureRequest) -> mock.MagicMock: patcher = mock.patch("bandersnatch.mirror.BandersnatchMirror") mirror: mock.MagicMock = patcher.start() def tearDown() -> None: patcher.stop() request.addfinalizer(tearDown) return mirror
def dag_folder(request: FixtureRequest) -> None: """Create dag folder and add finalizer to remove it again after running the test.""" folder = get_dags_folder() if not os.path.isdir(folder): os.mkdir(folder) def fin() -> None: shutil.rmtree(folder) request.addfinalizer(fin)
def sync_result(request: FixtureRequest) -> None: """Create sync-results and add finalizer to remove the folders after running the test.""" # For sync job test if not os.path.isdir(settings.SYNC_RESULTS_FOLDER): os.makedirs(settings.SYNC_RESULTS_FOLDER) def fin() -> None: shutil.rmtree(settings.SYNC_RESULTS_FOLDER) request.addfinalizer(fin)
def head_uc(request: FixtureRequest): request.instance.driver = uc.Chrome() def teardown(): request.instance.driver.save_screenshot(FAILED_SCREENSHOT_NAME) request.instance.driver.quit() request.addfinalizer(teardown) return request.instance.driver
def quilc(request: FixtureRequest) -> None: dock = docker.from_env() container = dock.containers.run( image="rigetti/quilc", command="-S", detach=True, ports={5555: 5555}, remove=True, ) request.addfinalizer(container.stop) return None
def fs(self, request: FixtureRequest, new_fs: "TestBuildHashEntryPath.FS") -> "TestBuildHashEntryPath.FS": new_fs.prev_dir = os.getcwd() def pop_lvl1(): os.chdir(new_fs.prev_dir) os.chdir(new_fs.level1.realpath()) request.addfinalizer(pop_lvl1) return new_fs
def start_server_process(request: FixtureRequest): proc = Process(target=start_app) proc.daemon = True proc.start() time.sleep(2) def fin(): proc.terminate() request.addfinalizer(fin)
def headless_uc(request: FixtureRequest): options = uc.ChromeOptions() options.headless = True request.instance.driver = uc.Chrome(options=options) def teardown(): request.instance.driver.sapipve_screenshot(FAILED_SCREENSHOT_NAME) request.instance.driver.quit() request.addfinalizer(teardown) return request.instance.driver
def fs(self, request: FixtureRequest, new_fs: "TestPathResolution.FS") -> "TestPathResolution.FS": new_fs.prev_dir = os.getcwd() def pop_lvl1(): os.chdir(new_fs.prev_dir) os.chdir(new_fs.level1.realpath()) new_fs.record_hashes() request.addfinalizer(pop_lvl1) return new_fs
def qvm(request: FixtureRequest) -> None: print("running qvm container") dock = docker.from_env() container = dock.containers.run(image="rigetti/qvm", command="-S", detach=True, ports={5000: 5000}, remove=True) # container = dock.containers.run(image="rigetti/qvm", command="-S", detach=True, # publish_all_ports=True, remove=True) request.addfinalizer(container.stop) return None
def enable_video(request: FixtureRequest, settings: Settings, name): def save(): for _ in range(60): try: open(f'{settings.video_dir}/{name}', 'rb') # noqa break except FileNotFoundError: time.sleep(1) allure.attach.file(f'{settings.video_dir}/{name}', attachment_type=allure.attachment_type.MP4) request.addfinalizer(save)
def tmp_folder(request: FixtureRequest) -> str: """Fixture to get path to data tmp folder for tests.""" folder = get_tmp_folder() if not os.path.isdir(folder): os.makedirs(folder) def fin() -> None: if os.path.isdir(folder): shutil.rmtree(folder) request.addfinalizer(fin) return folder
def browser(request: FixtureRequest, opencart_url: str, is_headless: bool, webdriver_logging: bool) -> webdriver: """ Launch browser and open page specified in --opencart_url command line option """ browser = request.config.getoption(name="--browser") driver = driver_factory(browser_name=browser, is_headless=is_headless, webdriver_logging=webdriver_logging) request.addfinalizer(driver.quit) driver.maximize_window() driver.get(url=opencart_url) return driver
def reset_registry(request: FixtureRequest) -> None: """Ensure everything has a clean registry.""" reset = {} for name in dir(static): obj = getattr(static, name) if hasattr(obj, '__registry__'): reset[obj] = getattr(obj, '__registry__').copy() def _fin(): for obj_, __registry__ in reset.items(): obj_.__registry__ = __registry__ request.addfinalizer(_fin)
def test_live_report(request: FixtureRequest, testdir): test_case_name = request.node.name def fin(): runner.invoke(syspath_sleuth.syspath_sleuth_main, ["-u"]) request.addfinalizer(finalizer=fin) runner = CliRunner() runner.invoke(syspath_sleuth.syspath_sleuth_main, ["-i"]) assert is_sleuth_active(), ( f"SysPathSleuth is not active, $SYSPATH_SLEUTH_KILL enabled?: " f"{os.getenv('SYSPATH_SLEUTH_KILL') is not None}") temp_test_py_file = testdir.makepyfile(""" import sys sys.path.append("yow") sys.path.insert(0, "yowsa") sys.path.remove("yow") sys.path.pop() sys.path.extend(["yow", "yowsa"]) """) result = testdir.runpython(temp_test_py_file) relevant_index = 0 regex = r"SysPathSleuth is installed in (system|user) site:" assert re.match(regex, result.outlines[relevant_index]) for index in range(1, len(result.outlines)): if "yow" in result.outlines[index]: relevant_index = index break regex = r"sys\.path\.append\(\'yow\',\) from .*%s\.py:2$" % test_case_name assert re.match(regex, result.outlines[relevant_index]) relevant_index += 1 regex = r"sys\.path\.insert\(0, \'yowsa\'\) from .*%s\.py:3$" % test_case_name assert re.match(regex, result.outlines[relevant_index]) relevant_index += 1 regex = r"sys\.path\.remove\(\'yow\',\) from .*%s\.py:4$" % test_case_name assert re.match(regex, result.outlines[relevant_index]) relevant_index += 1 regex = r"sys\.path\.pop\(\) from .*%s\.py:5$" % test_case_name assert re.match(regex, result.outlines[relevant_index]) relevant_index += 1 regex = r"sys\.path\.extend\(\[\'yow\', \'yowsa\'\],\) from .*%s\.py:6$" % test_case_name assert re.match(regex, result.outlines[relevant_index])
def user_id_folder(request: FixtureRequest) -> Tuple[str, str]: """Create return a test user folder and the fitting user_id as Tuple. A finalizer is added to remove the created user folder at the end of the test. """ user_id = "test-user" folder = create_user_folder(user_id) def fin() -> None: shutil.rmtree(folder) request.addfinalizer(fin) return folder, user_id
def mirror_mock(request: FixtureRequest) -> mock.MagicMock: patcher = mock.patch("bandersnatch.mirror.BandersnatchMirror") mirror: mock.MagicMock = patcher.start() # Mock the synchronize method too to avoid TypeError exceptions since methods are # by default replaced by MagicMock instances when AsyncMock is necessary. instance = mirror.return_value instance.synchronize = mock.AsyncMock(return_value={}) def tearDown() -> None: patcher.stop() request.addfinalizer(tearDown) return mirror
def spark_session(request: FixtureRequest) -> SparkSession: """ Create a pytest SparkSession fixture for testing. :param request: pytest FixtureRequest object :return: A SparkSession """ spark = (SparkSession.builder .master("local") .appName("test_pyspark_session") .getOrCreate()) # Teardown spark session between module tests request.addfinalizer(lambda: spark.stop()) return spark
def job_output(request: FixtureRequest) -> None: """Create job folder with output and add finalizer to remove the folders after running the test.""" job_results = os.path.join(settings.JOB_FOLDER, "result") if not os.path.isdir(job_results): os.makedirs(job_results) # Create empty file (mock job output) open(os.path.join(job_results, "sample-output.tif"), 'w').close() shutil.copyfile( os.path.join(get_test_data_folder(), "results_metadata.json"), os.path.join(job_results, "results_metadata.json")) def fin() -> None: shutil.rmtree(settings.JOB_FOLDER) request.addfinalizer(fin)
def user_folder(request: FixtureRequest) -> str: """Create and return a test user folder. The user id is set to 'test-user'. A finalizer is added to remove the created user folder at the end of the test. """ user_id = "test-user" folder = create_user_folder(user_id) def fin() -> None: shutil.rmtree(folder) request.addfinalizer(fin) return folder
def make_generic_classes( request: FixtureRequest, monkeypatch: MonkeyPatch, ) -> _MakeGenericClasses: def fin() -> None: for el in _need_generic: with suppress(AttributeError): delattr(el.cls, "__class_getitem__") def factory(django_version: Optional[_VersionSpec] = None) -> None: if django_version is not None: monkeypatch.setattr(patch, "VERSION", django_version) django_stubs_ext.monkeypatch() request.addfinalizer(fin) return factory