def shut_down_fork(forked, repo):
if forked:
cmd = ["mongo", "admin", "--eval", "db.shutdownServer()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(f'Deleting the test database failed, insert \"mongo admin --eval '
f'\"db.shutdownServer()\"\" into command line manually')
def sync_git(store, path):
"""Syncs the local documents via git."""
storedir, _ = os.path.split(path)
# get or update the storage
if os.path.isdir(storedir):
cmd = ["git", "pull"]
cwd = storedir
else:
cmd = ["git", "clone", store["url"], storedir]
cwd = None
subprocess.check_call(cmd, cwd=cwd)
def load_database(self, db):
"""Loads a database via mongoimport. Takes a database dict db."""
dbpath = dbpathname(db, self.rc)
for f in iglob(os.path.join(dbpath, "*.json")):
base, ext = os.path.splitext(os.path.split(f)[-1])
cmd = [
"mongoimport",
"--db",
db["name"],
"--collection",
base,
"--file",
f,
]
subprocess.check_call(cmd)
def rmtree(dirname):
"""Remove a directory, even if it has read-only files (Windows).
Git creates read-only files that must be removed on teardown. See
https://stackoverflow.com/questions/2656322 for more info.
Parameters
----------
dirname : str
Directory to be removed
"""
try:
shutil.rmtree(dirname)
except PermissionError:
if sys.platform == "win32":
subprocess.check_call(["del", "/F/S/Q", dirname], shell=True)
else:
raise
def is_alive(self):
"""Returns whether or not the client is alive and availabe to
send/recieve data.
"""
if self.client is None:
return False
elif ON_PYMONGO_V2:
return self.client.alive()
elif ON_PYMONGO_V3:
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
try:
subprocess.check_call(cmd)
alive = True
except subprocess.CalledProcessError:
alive = False
return alive
else:
return False
def dump_database(self, db):
"""Dumps a database dict via mongoexport."""
dbpath = dbpathname(db, self.rc)
os.makedirs(dbpath, exist_ok=True)
to_add = []
colls = self.client[db["name"]].collection_names(
include_system_collections=False)
for collection in colls:
f = os.path.join(dbpath, collection + ".json")
cmd = [
"mongoexport",
"--db",
db["name"],
"--collection",
collection,
"--out",
f,
]
subprocess.check_call(cmd)
to_add.append(os.path.join(db["path"], collection + ".json"))
return to_add
def deploy_git(rc, name, url, src="html", dst=None):
"""Loads a git database"""
targetdir = os.path.join(rc.deploydir, name)
# get or update the database
if os.path.isdir(targetdir):
cmd = ["git", "pull"]
cwd = targetdir
else:
cmd = ["git", "clone", url, targetdir]
cwd = None
subprocess.check_call(cmd, cwd=cwd)
# copy the files over
srcdir = os.path.join(rc.builddir, src)
dstdir = os.path.join(targetdir, dst) if dst else targetdir
copy_tree(srcdir, dstdir, verbose=1)
# commit everything
cmd = ["git", "add", "."]
subprocess.check_call(cmd, cwd=targetdir)
# commit
cmd = [
"git",
"commit",
"-m",
"regolith auto-deploy at {0}".format(time.time()),
]
try:
subprocess.check_call(cmd, cwd=targetdir)
except subprocess.CalledProcessError:
warn("Could not git commit to " + targetdir, RuntimeWarning)
# don't return, just in case...
# deploy!
cmd = ["git", "push"]
try:
subprocess.check_call(cmd, cwd=targetdir)
except subprocess.CalledProcessError:
warn("Could not git push from " + targetdir, RuntimeWarning)
return
def push_git(store, path):
"""Pushes the local documents via git."""
storedir, _ = os.path.split(path)
cmd = ["git", "add", "."]
subprocess.check_call(cmd, cwd=storedir)
cmd = ["git", "commit", "-m", "regolith auto-store commit"]
try:
subprocess.check_call(cmd, cwd=storedir)
except subprocess.CalledProcessError:
warn("Could not git commit to " + storedir, RuntimeWarning)
return
cmd = ["git", "push"]
try:
subprocess.check_call(cmd, cwd=storedir)
except subprocess.CalledProcessError:
warn("Could not git push from " + storedir, RuntimeWarning)
return
def make_mongodb():
"""A test fixutre that creates and destroys a git repo in a temporary
directory.
This will yield the path to the repo.
"""
cwd = os.getcwd()
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname":
"ERGS",
"databases": [{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
}],
"stores": [{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}],
"mongodbpath":
mongodbpath,
"backend":
"mongodb"
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
#cmd = ["mongod", "--dbpath", mongodbpath]
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"If on linux or mac, Mongod command failed to execute. If on windows, mongod has not been installed as \n"
"a service. In order to run mongodb tests, make sure to install the mongodb community edition\n"
"for your OS with the following link: https://docs.mongodb.com/manual/installation/"
)
yield False
return
# Write collection docs
for col_name, example in deepcopy(EXEMPLARS).items():
try:
client = MongoClient('localhost', serverSelectionTimeoutMS=2000)
client.server_info()
except Exception as e:
yield False
return
db = client['test']
col = db[col_name]
try:
if isinstance(example, list):
for doc in example:
doc['_id'].replace('.', '')
col.insert_many(example)
else:
example['_id'].replace('.', '')
col.insert_one(example)
except mongo_errors.DuplicateKeyError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
except mongo_errors.BulkWriteError:
print(
'Duplicate key error, check exemplars for duplicates if tests fail'
)
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
if not OUTPUT_FAKE_DB:
rmtree(repo)
def make_mongodb():
"""A test fixture that creates and destroys a git repo in a temporary
directory, as well as a mongo database.
This will yield the path to the repo.
"""
cwd = os.getcwd()
forked = False
name = "regolith_mongo_fake"
repo = os.path.join(tempfile.gettempdir(), name)
if os.path.exists(repo):
rmtree(repo)
subprocess.run(["git", "init", repo])
os.chdir(repo)
with open("README", "w") as f:
f.write("testing " + name)
mongodbpath = os.path.join(repo, 'dbs')
os.mkdir(mongodbpath)
with open("regolithrc.json", "w") as f:
json.dump(
{
"groupname": "ERGS",
"databases": [
{
"name": REGOLITH_MONGODB_NAME,
"url": 'localhost',
"path": repo,
"public": True,
"local": True,
"backend": "mongodb"
}
],
"stores": [
{
"name": "store",
"url": repo,
"path": repo,
"public": True,
}
],
"mongodbpath": mongodbpath,
},
f,
)
if os.name == 'nt':
# If on windows, the mongod command cannot be run with the fork or syslog options. Instead, it is installed as
# a service and the exceptions that would typically be log outputs are handled by the exception handlers below.
# In addition, the database must always be manually deleted from the windows mongo instance before running a
# fresh test.
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(
"Mongodb likely has not been installed as a service. In order to run mongodb tests, make sure\n"
"to install the mongodb community edition with the following link: \n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
cmd = ["mongostat", "--host", "localhost", "-n", "1"]
else:
cmd = ['mongod', '--fork', '--syslog', '--dbpath', mongodbpath]
forked = True
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print("If using linux or mac, Mongod command failed to execute. If using windows, the status of mongo could \n"
"not be retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with"
"\nthe following link:\n"
"https://docs.mongodb.com/manual/installation/")
yield False
return
try:
exemplars_to_mongo(REGOLITH_MONGODB_NAME)
except:
yield False
return
yield repo
cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"]
try:
subprocess.check_call(cmd, cwd=repo)
except subprocess.CalledProcessError:
print(f'Deleting the test database failed, insert \"mongo {REGOLITH_MONGODB_NAME} --eval '
f'\"db.dropDatabase()\"\" into command line manually')
shut_down_fork(forked, repo)
if not OUTPUT_FAKE_DB:
rmtree(repo)