def pip_install_artifact(request):
wheel = None
test_env = tempfile.mkdtemp(prefix="test-env")
def create_wheel_and_install(artifact_ref, include_requirements=True):
nonlocal wheel
artifact_path = download_artifact(artifact_ref)
wheel = artifact_to_wheel(artifact_path,
include_requirements=include_requirements)
subprocess.run(['virtualenv', test_env], check=True)
if sys.platform.startswith('win'):
site_packages = os.path.join(test_env, 'Lib', 'site-packages')
code = f"{test_env}\\Scripts\\activate & pip install {wheel.filename}"
subprocess.run(code, check=True, shell=True)
else:
site_packages = glob.glob(
os.path.join(test_env, 'lib', 'python*', 'site-packages'))[0]
code = f"source {test_env}/bin/activate; pip install {wheel.filename}"
# uncomment the following when we handle dependencies
#import_tests = os.path.join(wheel.basedir, 'info', 'test', 'run_test.py')
#if os.path.isfile(import_tests):
# code += f"; python {import_tests}"
subprocess.run(["bash", "-c", code], check=True)
return wheel, test_env, site_packages
yield create_wheel_and_install
if wheel is not None:
wheel.clean()
rmtree(test_env, force=True)
wheels = glob.glob(os.path.join(os.path.dirname(__file__), "*.whl"))
for w in wheels:
os.remove(w)
def make_mongodb():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname":
"ERGS",
"databases": [{
"name": "test",
"url": 'localhost',
"path": repo,
"public": True,
"local": False,
}],
"stores": [{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}],
"mongodbpath":
mongodbpath,
"backend":
"mongodb"
},
f,
)
subprocess.run(['mongod', '--fork', '--syslog', '--dbpath', mongodbpath])
# Write collection docs
for col_name, example in deepcopy(EXEMPLARS).items():
client = MongoClient('localhost', serverSelectionTimeoutMS=2000)
db = client['test']
col = db[col_name]
if isinstance(example, list):
for doc in example:
doc['_id'].replace('.', '')
col.insert_many(example)
else:
example['_id'].replace('.', '')
col.insert_one(example)
yield repo
if not OUTPUT_FAKE_DB:
rmtree(repo)
def make_bad_db():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_fake_bad"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": "test",
"url": repo,
"public": True,
"path": "db",
"local": True,
}
],
},
f,
)
os.mkdir("db")
# Write collection docs
for coll, example in deepcopy(EXEMPLARS).items():
if isinstance(example, list):
d = {dd["_id"]: dd for dd in example}
else:
d = {example["_id"]: example}
d.update({"bad": {"_id": "bad", "bad": True}})
if coll == "presentations":
d.update(
{
"bad_inst": {
"_id": "bad_inst",
"institution": "noinstitution",
"department": "nodept",
}
}
)
dump_yaml("db/{}.yaml".format(coll), d)
subprocess.run(["git", "add", "."])
subprocess.run(["git", "commit", "-am", "Initial readme"])
yield repo
os.chdir(cwd)
rmtree(repo)
def pip_install_artifact_tree(request):
wheels = {}
test_env = tempfile.mkdtemp(prefix="test-env")
def create_wheels_and_install(artifact_ref,
include_requirements=True,
skip_python=False,
fatten=False,
skipped_deps=None):
nonlocal wheels
seen = artifact_ref_dependency_tree_to_wheels(
artifact_ref,
seen=wheels,
config=Config(
skip_python=skip_python,
include_requirements=include_requirements,
fatten=fatten,
subdir=PLATFORM_TO_SUBDIR[sys.platform],
),
)
if fatten:
wheels = fatten_from_seen(seen, skipped_deps=skipped_deps)
subprocess.run(['virtualenv', test_env], check=True)
wheel_filenames = " ".join(
reversed([w.filename for w in wheels.values() if w is not None]))
if sys.platform.startswith('win'):
site_packages = os.path.join(test_env, 'Lib', 'site-packages')
code = f"{test_env}\\Scripts\\activate & pip install {wheel_filenames}"
print("Running:\n " + code)
subprocess.run(code, check=True, shell=True)
else:
site_packages = glob.glob(
os.path.join(test_env, 'lib', 'python*', 'site-packages'))[0]
code = f"source {test_env}/bin/activate; pip install {wheel_filenames}"
# uncomment the following when we handle dependencies
#import_tests = os.path.join(wheel.basedir, 'info', 'test', 'run_test.py')
#if os.path.isfile(import_tests):
# code += f"; python {import_tests}"
print("Running:\n " + code)
subprocess.run(["bash", "-c", code], check=True)
return wheels, test_env, site_packages
yield create_wheels_and_install
for wheel in wheels.values():
if wheel is None:
continue
wheel.clean()
rmtree(test_env, force=True)
wheel_names = glob.glob(os.path.join(os.path.dirname(__file__), "*.whl"))
for w in wheel_names:
os.remove(w)
def make_db():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": "test",
"url": repo,
"public": True,
"path": "db",
"local": True,
"backend": "filesystem"
}
],
"stores": [
{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}
],
},
f,
)
fspath = os.path.join(repo, 'db')
os.mkdir(fspath)
exemplars_to_fs(fspath)
subprocess.run(["git", "add", "."])
subprocess.run(["git", "commit", "-am", "Initial readme"])
yield repo
os.chdir(cwd)
if not OUTPUT_FAKE_DB:
rmtree(repo)
def pip_install_artifact_tree(request):
wheels = {}
test_env = tempfile.mkdtemp(prefix="test-env")
def create_wheels_and_install(artifact_ref,
include_requirements=True,
skip_python=False):
nonlocal wheels
subdir = PLATFORM_TO_SUBDIR[sys.platform]
artifact_ref_dependency_tree_to_wheels(
artifact_ref,
seen=wheels,
subdir=subdir,
include_requirements=include_requirements,
skip_python=skip_python,
)
subprocess.run(['virtualenv', test_env], check=True)
wheel_filenames = " ".join(
reversed([w.filename for w in wheels.values() if w is not None]))
if skip_python:
# FIXME: remove this case as part of resolving
# https://github.com/regro/conda-press/issues/17
site_packages = "fake!"
elif sys.platform.startswith('win'):
site_packages = os.path.join(test_env, 'Lib', 'site-packages')
code = f"{test_env}\\Scripts\\activate & pip install {wheel_filenames}"
subprocess.run(code, check=True, shell=True)
else:
site_packages = glob.glob(
os.path.join(test_env, 'lib', 'python*', 'site-packages'))[0]
code = f"source {test_env}/bin/activate; pip install {wheel_filenames}"
# uncomment the following when we handle dependencies
#import_tests = os.path.join(wheel.basedir, 'info', 'test', 'run_test.py')
#if os.path.isfile(import_tests):
# code += f"; python {import_tests}"
subprocess.run(["bash", "-c", code], check=True)
return wheels, test_env, site_packages
yield create_wheels_and_install
for wheel in wheels.values():
if wheel is None:
continue
wheel.clean()
rmtree(test_env, force=True)
wheel_names = glob.glob(os.path.join(os.path.dirname(__file__), "*.whl"))
for w in wheel_names:
os.remove(w)
def make_mongodb():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname":
"ERGS",
"databases": [{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
}],
"stores": [{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}],
"mongodbpath":
mongodbpath,
"backend":
"mongodb"
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
#cmd = ["mongod", "--dbpath", mongodbpath]
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
# Write collection docs
for col_name, example in deepcopy(EXEMPLARS).items():
try:
client = MongoClient('localhost', serverSelectionTimeoutMS=2000)
client.server_info()
except Exception as e:
yield False
return
db = client['test']
col = db[col_name]
try:
if isinstance(example, list):
for doc in example:
doc['_id'].replace('.', '')
col.insert_many(example)
else:
example['_id'].replace('.', '')
col.insert_one(example)
except mongo_errors.DuplicateKeyError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
except mongo_errors.BulkWriteError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
if not OUTPUT_FAKE_DB:
rmtree(repo)
def make_mongodb():
"""A test fixture that creates and destroys a git repo in a temporary
directory, as well as a mongo database.
This will yield the path to the repo.
"""
cwd = os.getcwd()
forked = False
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
"backend": "mongodb"
}
],
"stores": [
{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}
],
"mongodbpath": mongodbpath,
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"Mongodb likely has not been installed as a service. In order to run mongodb tests, make sure\n"
"to install the mongodb community edition with the following link: \n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
forked = True
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print("If using linux or mac, Mongod command failed to execute. If using windows, the status of mongo could \n"
"not be retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with"
"\nthe following link:\n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
try:
exemplars_to_mongo(REGOLITH_MONGODB_NAME)
except:
yield False
return
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
shut_down_fork(forked, repo)
if not OUTPUT_FAKE_DB:
rmtree(repo)
