def spark_session(request: FixtureRequest) -> SparkSession:
# make sure env variables are set correctly
if "SPARK_HOME" not in os.environ:
os.environ["SPARK_HOME"] = "/usr/local/opt/spark"
clean_spark_dir()
master = "local[2]"
session = (
SparkSession.builder.appName(
f"pytest-pyspark-local-testing-{get_random_string(4)}"
)
.master(master)
.config("spark.ui.showConsoleProgress", "false")
.config("spark.sql.shuffle.partitions", "2")
.config("spark.default.parallelism", "4")
.config("spark.driver.bindAddress", "127.0.0.1")
.config("spark.sql.broadcastTimeout", "2400")
.enableHiveSupport()
.getOrCreate()
)
request.addfinalizer(lambda: clean_close(session))
quiet_py4j()
return session
def database(app: Flask, request: FixtureRequest) -> SQLAlchemy:
if os.path.exists(TEST_DATABASE_PATH):
os.unlink(TEST_DATABASE_PATH)
db.app = app
db.create_all()
# Bypass pysqlite's broken transactions (see https://bit.ly/2DKiixa).
# pylint:disable=unused-argument
# pylint:disable=unused-variable
@event.listens_for(db.engine, 'connect')
def do_connect(connection: Connection, *args) -> None:
connection.isolation_level = None
# pylint:disable=unused-variable
@event.listens_for(db.engine, 'begin')
def do_begin(connection: Connection) -> None:
connection.execute('BEGIN')
def teardown() -> None:
db.drop_all()
os.unlink(TEST_DATABASE_PATH)
request.addfinalizer(teardown)
return db
def spark_session(request: FixtureRequest) -> SparkSession:
# make sure env variables are set correctly
if 'SPARK_HOME' not in os.environ:
os.environ['SPARK_HOME'] = '/usr/local/opt/spark'
clean_spark_dir()
master = "local[2]"
# master: str = "spark://localhost:7077"
if not path.exists("/Applications/Docker.app"):
master = "local[2]"
print(f"++++++ Running on local spark: {master} ++++")
else:
print(f"++++++ Running on docker spark: {master} ++++")
session = SparkSession.builder.appName(f"pytest-pyspark-local-testing-{get_random_string(4)}"). \
master(master). \
config("spark.ui.showConsoleProgress", "false"). \
config("spark.sql.shuffle.partitions", "2"). \
config("spark.default.parallelism", "4"). \
config("spark.sql.broadcastTimeout", "2400"). \
enableHiveSupport(). \
getOrCreate()
request.addfinalizer(lambda: clean_close(session))
quiet_py4j()
return session
def initialize_tests(request: FixtureRequest) -> None:
initializer(
["eventual_tortoise.relation"],
db_url="sqlite://:memory:",
app_label="default",
)
request.addfinalizer(finalizer)
def never_sleep(request: FixtureRequest) -> None:
patcher = mock.patch("asyncio.sleep", _nosleep)
patcher.start()
def tearDown() -> None:
patcher.stop()
request.addfinalizer(tearDown)
def stop_std_logging(request: FixtureRequest, capfd: CaptureFixture) -> None:
patcher = mock.patch("bandersnatch.log.setup_logging")
patcher.start()
def tearDown() -> None:
patcher.stop()
request.addfinalizer(tearDown)
def job_folder(request: FixtureRequest) -> None:
"""Create job folder and add finalizer to remove the folders after running the test."""
if not os.path.isdir(settings.JOB_FOLDER):
os.makedirs(settings.JOB_FOLDER)
def fin() -> None:
shutil.rmtree(settings.JOB_FOLDER)
request.addfinalizer(fin)
def selenium(request: FixtureRequest) -> webdriver.Remote:
if request.config.getoption('--local-chrome'):
driver = initialize_local_chrome()
request.addfinalizer(lambda: deinitialize_selenium(driver))
return driver
driver = request.getfixturevalue('selenium') # type: webdriver.Remote
yield driver
# https://github.com/SeleniumHQ/docker-selenium/issues/87#issuecomment-286231268
driver.close()
def logging_mock(request: FixtureRequest) -> mock.MagicMock:
patcher = mock.patch("logging.config.fileConfig")
logger: mock.MagicMock = patcher.start()
def tearDown() -> None:
patcher.stop()
request.addfinalizer(tearDown)
return logger
def selenium(request: FixtureRequest) -> webdriver.Remote:
if request.config.getoption('--local-chrome'):
driver = initialize_local_chrome()
request.addfinalizer(lambda: deinitialize_selenium(driver))
return driver
driver = request.getfixturevalue('selenium') # type: webdriver.Remote
yield driver
# https://github.com/SeleniumHQ/docker-selenium/issues/87#issuecomment-286231268
driver.close()
def mirror_mock(request: FixtureRequest) -> mock.MagicMock:
patcher = mock.patch("bandersnatch.mirror.BandersnatchMirror")
mirror: mock.MagicMock = patcher.start()
def tearDown() -> None:
patcher.stop()
request.addfinalizer(tearDown)
return mirror
def dag_folder(request: FixtureRequest) -> None:
"""Create dag folder and add finalizer to remove it again after running the test."""
folder = get_dags_folder()
if not os.path.isdir(folder):
os.mkdir(folder)
def fin() -> None:
shutil.rmtree(folder)
request.addfinalizer(fin)
def sync_result(request: FixtureRequest) -> None:
"""Create sync-results and add finalizer to remove the folders after running the test."""
# For sync job test
if not os.path.isdir(settings.SYNC_RESULTS_FOLDER):
os.makedirs(settings.SYNC_RESULTS_FOLDER)
def fin() -> None:
shutil.rmtree(settings.SYNC_RESULTS_FOLDER)
request.addfinalizer(fin)
def head_uc(request: FixtureRequest):
request.instance.driver = uc.Chrome()
def teardown():
request.instance.driver.save_screenshot(FAILED_SCREENSHOT_NAME)
request.instance.driver.quit()
request.addfinalizer(teardown)
return request.instance.driver
def quilc(request: FixtureRequest) -> None:
dock = docker.from_env()
container = dock.containers.run(
image="rigetti/quilc",
command="-S",
detach=True,
ports={5555: 5555},
remove=True,
)
request.addfinalizer(container.stop)
return None
def fs(self, request: FixtureRequest,
new_fs: "TestBuildHashEntryPath.FS") -> "TestBuildHashEntryPath.FS":
new_fs.prev_dir = os.getcwd()
def pop_lvl1():
os.chdir(new_fs.prev_dir)
os.chdir(new_fs.level1.realpath())
request.addfinalizer(pop_lvl1)
return new_fs
def start_server_process(request: FixtureRequest):
proc = Process(target=start_app)
proc.daemon = True
proc.start()
time.sleep(2)
def fin():
proc.terminate()
request.addfinalizer(fin)
def headless_uc(request: FixtureRequest):
options = uc.ChromeOptions()
options.headless = True
request.instance.driver = uc.Chrome(options=options)
def teardown():
request.instance.driver.sapipve_screenshot(FAILED_SCREENSHOT_NAME)
request.instance.driver.quit()
request.addfinalizer(teardown)
return request.instance.driver
def fs(self, request: FixtureRequest,
new_fs: "TestPathResolution.FS") -> "TestPathResolution.FS":
new_fs.prev_dir = os.getcwd()
def pop_lvl1():
os.chdir(new_fs.prev_dir)
os.chdir(new_fs.level1.realpath())
new_fs.record_hashes()
request.addfinalizer(pop_lvl1)
return new_fs
def qvm(request: FixtureRequest) -> None:
print("running qvm container")
dock = docker.from_env()
container = dock.containers.run(image="rigetti/qvm",
command="-S",
detach=True,
ports={5000: 5000},
remove=True)
# container = dock.containers.run(image="rigetti/qvm", command="-S", detach=True,
# publish_all_ports=True, remove=True)
request.addfinalizer(container.stop)
return None
def enable_video(request: FixtureRequest, settings: Settings, name):
def save():
for _ in range(60):
try:
open(f'{settings.video_dir}/{name}', 'rb') # noqa
break
except FileNotFoundError:
time.sleep(1)
allure.attach.file(f'{settings.video_dir}/{name}',
attachment_type=allure.attachment_type.MP4)
request.addfinalizer(save)
def tmp_folder(request: FixtureRequest) -> str:
"""Fixture to get path to data tmp folder for tests."""
folder = get_tmp_folder()
if not os.path.isdir(folder):
os.makedirs(folder)
def fin() -> None:
if os.path.isdir(folder):
shutil.rmtree(folder)
request.addfinalizer(fin)
return folder
def browser(request: FixtureRequest, opencart_url: str, is_headless: bool,
webdriver_logging: bool) -> webdriver:
"""
Launch browser and open page specified in --opencart_url command line option
"""
browser = request.config.getoption(name="--browser")
driver = driver_factory(browser_name=browser,
is_headless=is_headless,
webdriver_logging=webdriver_logging)
request.addfinalizer(driver.quit)
driver.maximize_window()
driver.get(url=opencart_url)
return driver
def reset_registry(request: FixtureRequest) -> None:
"""Ensure everything has a clean registry."""
reset = {}
for name in dir(static):
obj = getattr(static, name)
if hasattr(obj, '__registry__'):
reset[obj] = getattr(obj, '__registry__').copy()
def _fin():
for obj_, __registry__ in reset.items():
obj_.__registry__ = __registry__
request.addfinalizer(_fin)
def test_live_report(request: FixtureRequest, testdir):
test_case_name = request.node.name
def fin():
runner.invoke(syspath_sleuth.syspath_sleuth_main, ["-u"])
request.addfinalizer(finalizer=fin)
runner = CliRunner()
runner.invoke(syspath_sleuth.syspath_sleuth_main, ["-i"])
assert is_sleuth_active(), (
f"SysPathSleuth is not active, $SYSPATH_SLEUTH_KILL enabled?: "
f"{os.getenv('SYSPATH_SLEUTH_KILL') is not None}")
temp_test_py_file = testdir.makepyfile("""
import sys
sys.path.append("yow")
sys.path.insert(0, "yowsa")
sys.path.remove("yow")
sys.path.pop()
sys.path.extend(["yow", "yowsa"])
""")
result = testdir.runpython(temp_test_py_file)
relevant_index = 0
regex = r"SysPathSleuth is installed in (system|user) site:"
assert re.match(regex, result.outlines[relevant_index])
for index in range(1, len(result.outlines)):
if "yow" in result.outlines[index]:
relevant_index = index
break
regex = r"sys\.path\.append\(\'yow\',\) from .*%s\.py:2$" % test_case_name
assert re.match(regex, result.outlines[relevant_index])
relevant_index += 1
regex = r"sys\.path\.insert\(0, \'yowsa\'\) from .*%s\.py:3$" % test_case_name
assert re.match(regex, result.outlines[relevant_index])
relevant_index += 1
regex = r"sys\.path\.remove\(\'yow\',\) from .*%s\.py:4$" % test_case_name
assert re.match(regex, result.outlines[relevant_index])
relevant_index += 1
regex = r"sys\.path\.pop\(\) from .*%s\.py:5$" % test_case_name
assert re.match(regex, result.outlines[relevant_index])
relevant_index += 1
regex = r"sys\.path\.extend\(\[\'yow\', \'yowsa\'\],\) from .*%s\.py:6$" % test_case_name
assert re.match(regex, result.outlines[relevant_index])
def user_id_folder(request: FixtureRequest) -> Tuple[str, str]:
"""Create return a test user folder and the fitting user_id as Tuple.
A finalizer is added to remove the created user folder at the end of the test.
"""
user_id = "test-user"
folder = create_user_folder(user_id)
def fin() -> None:
shutil.rmtree(folder)
request.addfinalizer(fin)
return folder, user_id
def mirror_mock(request: FixtureRequest) -> mock.MagicMock:
patcher = mock.patch("bandersnatch.mirror.BandersnatchMirror")
mirror: mock.MagicMock = patcher.start()
# Mock the synchronize method too to avoid TypeError exceptions since methods are
# by default replaced by MagicMock instances when AsyncMock is necessary.
instance = mirror.return_value
instance.synchronize = mock.AsyncMock(return_value={})
def tearDown() -> None:
patcher.stop()
request.addfinalizer(tearDown)
return mirror
def spark_session(request: FixtureRequest) -> SparkSession:
"""
Create a pytest SparkSession fixture for testing.
:param request: pytest FixtureRequest object
:return: A SparkSession
"""
spark = (SparkSession.builder
.master("local")
.appName("test_pyspark_session")
.getOrCreate())
# Teardown spark session between module tests
request.addfinalizer(lambda: spark.stop())
return spark
def job_output(request: FixtureRequest) -> None:
"""Create job folder with output and add finalizer to remove the folders after running the test."""
job_results = os.path.join(settings.JOB_FOLDER, "result")
if not os.path.isdir(job_results):
os.makedirs(job_results)
# Create empty file (mock job output)
open(os.path.join(job_results, "sample-output.tif"), 'w').close()
shutil.copyfile(
os.path.join(get_test_data_folder(), "results_metadata.json"),
os.path.join(job_results, "results_metadata.json"))
def fin() -> None:
shutil.rmtree(settings.JOB_FOLDER)
request.addfinalizer(fin)
def user_folder(request: FixtureRequest) -> str:
"""Create and return a test user folder.
The user id is set to 'test-user'.
A finalizer is added to remove the created user folder at the end of the test.
"""
user_id = "test-user"
folder = create_user_folder(user_id)
def fin() -> None:
shutil.rmtree(folder)
request.addfinalizer(fin)
return folder
def make_generic_classes(
request: FixtureRequest,
monkeypatch: MonkeyPatch,
) -> _MakeGenericClasses:
def fin() -> None:
for el in _need_generic:
with suppress(AttributeError):
delattr(el.cls, "__class_getitem__")
def factory(django_version: Optional[_VersionSpec] = None) -> None:
if django_version is not None:
monkeypatch.setattr(patch, "VERSION", django_version)
django_stubs_ext.monkeypatch()
request.addfinalizer(fin)
return factory
