def make_mongodb():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname":
"ERGS",
"databases": [{
"name": "test",
"url": 'localhost',
"path": repo,
"public": True,
"local": False,
}],
"stores": [{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}],
"mongodbpath":
mongodbpath,
"backend":
"mongodb"
},
f,
)
subprocess.run(['mongod', '--fork', '--syslog', '--dbpath', mongodbpath])
# Write collection docs
for col_name, example in deepcopy(EXEMPLARS).items():
client = MongoClient('localhost', serverSelectionTimeoutMS=2000)
db = client['test']
col = db[col_name]
if isinstance(example, list):
for doc in example:
doc['_id'].replace('.', '')
col.insert_many(example)
else:
example['_id'].replace('.', '')
col.insert_one(example)
yield repo
if not OUTPUT_FAKE_DB:
rmtree(repo)
def run_test_yaml_migration(m,
*,
inp,
output,
kwargs,
prb,
mr_out,
tmpdir,
should_filter=False):
os.makedirs(os.path.join(tmpdir, "recipe"), exist_ok=True)
with open(os.path.join(tmpdir, "recipe", "meta.yaml"), "w") as f:
f.write(inp)
with indir(tmpdir):
subprocess.run(["git", "init"])
# Load the meta.yaml (this is done in the graph)
try:
pmy = parse_meta_yaml(inp)
except Exception:
pmy = {}
if pmy:
pmy["version"] = pmy["package"]["version"]
pmy["req"] = set()
for k in ["build", "host", "run"]:
pmy["req"] |= set(pmy.get("requirements", {}).get(k, set()))
try:
pmy["meta_yaml"] = parse_meta_yaml(inp)
except Exception:
pmy["meta_yaml"] = {}
pmy["raw_meta_yaml"] = inp
pmy.update(kwargs)
assert m.filter(pmy) is should_filter
if should_filter:
return
mr = m.migrate(os.path.join(tmpdir, "recipe"), pmy)
assert mr_out == mr
pmy.update(PRed=[frozen_to_json_friendly(mr)])
with open(os.path.join(tmpdir, "recipe/meta.yaml")) as f:
actual_output = f.read()
assert actual_output == output
assert os.path.exists(
os.path.join(tmpdir, ".ci_support/migrations/hi.yaml"))
with open(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml")) as f:
saved_migration = f.read()
assert saved_migration == m.yaml_contents
def test_builder(bm, make_db):
repo = make_db
os.chdir(repo)
if bm == "figure":
prep_figure()
if bm == "html":
os.makedirs("templates/static", exist_ok=True)
if bm == "reimb":
subprocess.run(
["regolith", "build", bm, "--no-pdf", "--people", "scopatz"],
check=True,
cwd=repo)
else:
subprocess.run(["regolith", "build", bm, "--no-pdf"],
check=True,
cwd=repo)
os.chdir(os.path.join(repo, "_build", bm))
expected_base = os.path.join(os.path.dirname(__file__), "outputs")
for root, dirs, files in os.walk("."):
for file in files:
if file in os.listdir(os.path.join(expected_base, bm, root)):
fn1 = os.path.join(repo, "_build", bm, root, file)
if bm == "reimb":
actual = openpyxl.load_workbook(fn1)["T&B"]
actual = [str(actual[b]) for b in xls_check]
else:
with open(fn1, "r") as f:
actual = f.read()
fn2 = os.path.join(expected_base, bm, root, file)
if bm == "reimb":
expected = openpyxl.load_workbook(fn2)["T&B"]
expected = [str(expected[b]) for b in xls_check]
else:
with open(fn2, "r") as f:
expected = f.read()
# Skip because of a date time in
if file != "rss.xml":
# Fixme proper fix for testing hard coded filepaths on windows
if os.name == "nt":
if "tmp" not in expected:
if "../.." not in expected:
assert expected == actual
else:
assert expected == actual
def make_bad_db():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_fake_bad"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": "test",
"url": repo,
"public": True,
"path": "db",
"local": True,
}
],
},
f,
)
os.mkdir("db")
# Write collection docs
for coll, example in deepcopy(EXEMPLARS).items():
if isinstance(example, list):
d = {dd["_id"]: dd for dd in example}
else:
d = {example["_id"]: example}
d.update({"bad": {"_id": "bad", "bad": True}})
if coll == "presentations":
d.update(
{
"bad_inst": {
"_id": "bad_inst",
"institution": "noinstitution",
"department": "nodept",
}
}
)
dump_yaml("db/{}.yaml".format(coll), d)
subprocess.run(["git", "add", "."])
subprocess.run(["git", "commit", "-am", "Initial readme"])
yield repo
os.chdir(cwd)
rmtree(repo)
def make_db():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": "test",
"url": repo,
"public": True,
"path": "db",
"local": True,
"backend": "filesystem"
}
],
"stores": [
{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}
],
},
f,
)
fspath = os.path.join(repo, 'db')
os.mkdir(fspath)
exemplars_to_fs(fspath)
subprocess.run(["git", "add", "."])
subprocess.run(["git", "commit", "-am", "Initial readme"])
yield repo
os.chdir(cwd)
if not OUTPUT_FAKE_DB:
rmtree(repo)
def test_validate_bad(make_bad_db):
repo = make_bad_db
os.chdir(repo)
out = subprocess.run(["regolith", "validate"], check=False).out
assert "Errors found in " in out
assert "NO ERRORS IN DBS" not in out
def test_validate(make_db):
repo = make_db
os.chdir(repo)
out = subprocess.run(["regolith", "validate"], check=False).out
assert "NO ERRORS IN DBS" in out
def make_mongodb():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname":
"ERGS",
"databases": [{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
}],
"stores": [{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}],
"mongodbpath":
mongodbpath,
"backend":
"mongodb"
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
#cmd = ["mongod", "--dbpath", mongodbpath]
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
# Write collection docs
for col_name, example in deepcopy(EXEMPLARS).items():
try:
client = MongoClient('localhost', serverSelectionTimeoutMS=2000)
client.server_info()
except Exception as e:
yield False
return
db = client['test']
col = db[col_name]
try:
if isinstance(example, list):
for doc in example:
doc['_id'].replace('.', '')
col.insert_many(example)
else:
example['_id'].replace('.', '')
col.insert_one(example)
except mongo_errors.DuplicateKeyError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
except mongo_errors.BulkWriteError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
if not OUTPUT_FAKE_DB:
rmtree(repo)
def test_builder(bm, db_src, make_db, make_mongodb):
# FIXME: Somehow the mongo backend failed to build figure
if db_src == "mongo" and bm == "figure":
return
if db_src == "fs":
repo = make_db
elif db_src == "mongo":
repo = make_mongodb
else:
raise ValueError("Unknown database source: {}".format(db_src))
os.chdir(repo)
if bm == "figure":
prep_figure()
if bm == "html":
os.makedirs("templates/static", exist_ok=True)
if bm == "reimb" or bm == "recent-collabs":
subprocess.run(
["regolith", "build", bm, "--no-pdf", "--people", "scopatz"],
check=True,
cwd=repo)
elif bm == "annual-activity":
subprocess.run([
"regolith", "build", bm, "--no-pdf", "--people", "sbillinge",
"--from", "2017-04-01"
],
check=True,
cwd=repo)
else:
subprocess.run(["regolith", "build", bm, "--no-pdf"],
check=True,
cwd=repo)
os.chdir(os.path.join(repo, "_build", bm))
expected_base = os.path.join(os.path.dirname(__file__), "outputs")
for root, dirs, files in os.walk("."):
for file in files:
if file in os.listdir(os.path.join(expected_base, bm, root)):
fn1 = os.path.join(repo, "_build", bm, root, file)
if bm == "reimb":
actual = openpyxl.load_workbook(fn1)["T&B"]
actual = [str(actual[b]) for b in xls_check]
elif bm == "recent-collabs":
if 'nsf' in fn1:
sheet = "NSF COA Template"
else:
sheet = "Collaborator Template"
actual = openpyxl.load_workbook(fn1)[sheet]
actual = [
str(actual[cell]) for cell in recent_collabs_xlsx_check
]
else:
with open(fn1, "r") as f:
actual = f.read()
fn2 = os.path.join(expected_base, bm, root, file)
if bm == "reimb":
expected = openpyxl.load_workbook(fn2)["T&B"]
expected = [str(expected[b]) for b in xls_check]
elif bm == "recent-collabs":
if 'nsf' in fn2:
sheet = "NSF COA Template"
else:
sheet = "Collaborator Template"
expected = openpyxl.load_workbook(fn2)[sheet]
expected = [
str(expected[cell])
for cell in recent_collabs_xlsx_check
]
else:
with open(fn2, "r") as f:
expected = f.read()
# Skip because of a date time in
if file != "rss.xml":
if file.endswith('.html') or file.endswith('.tex'):
assert is_same(expected, actual, ['../..', 'tmp'])
else:
assert expected == actual
def run(self, cmd):
"""Run command in build dir"""
subprocess.run(cmd, cwd=self.bldir, check=True)
def make_mongodb():
"""A test fixture that creates and destroys a git repo in a temporary
directory, as well as a mongo database.
This will yield the path to the repo.
"""
cwd = os.getcwd()
forked = False
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
"backend": "mongodb"
}
],
"stores": [
{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}
],
"mongodbpath": mongodbpath,
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"Mongodb likely has not been installed as a service. In order to run mongodb tests, make sure\n"
"to install the mongodb community edition with the following link: \n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
forked = True
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print("If using linux or mac, Mongod command failed to execute. If using windows, the status of mongo could \n"
"not be retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with"
"\nthe following link:\n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
try:
exemplars_to_mongo(REGOLITH_MONGODB_NAME)
except:
yield False
return
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
shut_down_fork(forked, repo)
if not OUTPUT_FAKE_DB:
rmtree(repo)
def test_builder(bm, db_src, make_db, make_mongodb, monkeypatch):
# FIXME: Somehow the mongo backend failed to build figure
# FIXME: now fs is failing to build figure
# if db_src == "mongo" and bm == "figure":
if bm == "figure":
return
if db_src == "fs":
repo = make_db
elif db_src == "mongo":
if make_mongodb is False:
pytest.skip("Mongoclient failed to start")
else:
repo = make_mongodb
else:
raise ValueError("Unknown database source: {}".format(db_src))
os.chdir(repo)
if bm == "figure":
prep_figure()
if bm == "internalhtml":
# for some reason the mocking of the crossref call doesn't work when the
# test is run using subprocess, so skip in this case.
# the functionality is fully tested in test_builder_python
pytest.skip("mocking of Crossref not working with subprocess")
if bm == "html":
os.makedirs("templates/static", exist_ok=True)
if bm == "reimb" or bm == "recent-collabs":
subprocess.run(
["regolith", "build", bm, "--no-pdf", "--people", "scopatz"],
check=True,
cwd=repo)
elif bm == "annual-activity":
subprocess.run([
"regolith", "build", bm, "--no-pdf", "--people", "sbillinge",
"--from", "2017-04-01"
],
check=True,
cwd=repo)
elif bm == "grantreport":
main([
"build", bm, "--no-pdf", "--grant", "SymPy-1.1", "--from",
"2017-04-01", "--to", "2018-03-31"
])
else:
subprocess.run(["regolith", "build", bm, "--no-pdf"],
check=True,
cwd=repo)
os.chdir(os.path.join(repo, "_build", bm))
expected_base = os.path.join(os.path.dirname(__file__), "outputs")
for root, dirs, files in os.walk("."):
for file in files:
if file in os.listdir(os.path.join(expected_base, bm, root)):
fn1 = os.path.join(repo, "_build", bm, root, file)
if bm == "reimb":
actual = openpyxl.load_workbook(fn1)["T&B"]
actual = [str(actual[b]) for b in xls_check]
elif bm == "recent-collabs":
if 'nsf' in fn1:
sheet = "NSF COA Template"
else:
sheet = "Collaborators"
actual = openpyxl.load_workbook(fn1)[sheet]
actual = [
str(actual[cell]) for cell in recent_collabs_xlsx_check
]
else:
with open(fn1, "r") as f:
actual = f.read()
fn2 = os.path.join(expected_base, bm, root, file)
if bm == "reimb":
expected = openpyxl.load_workbook(fn2)["T&B"]
expected = [str(expected[b]) for b in xls_check]
elif bm == "recent-collabs":
if 'nsf' in fn2:
sheet = "NSF COA Template"
else:
sheet = "Collaborators"
expected = openpyxl.load_workbook(fn2)[sheet]
expected = [
str(expected[cell])
for cell in recent_collabs_xlsx_check
]
else:
with open(fn2, "r") as f:
expected = f.read()
# Skip because of a date time in
if file != "rss.xml":
if file.endswith('.html') or file.endswith('.tex'):
if not is_same(expected, actual, ['../..', 'tmp']):
assert actual == expected
else:
assert actual == expected
