def admin_auth(project_name):
project_dir = os.path.join(quesadiya.get_projects_path(), project_name)
db_interface = factory.get_projectdb_interface(project_dir)
admin_name, admin_password = ask_admin_info()
auth = db_interface.admin_authentication(admin_name=admin_name,
admin_password=admin_password)
return auth
def test_default_action(self):
"""Test the command creates a project."""
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.jsonl"],
input='1234\n1234\n')
assert r.exception is None
# check the existence of project folder
assert os.path.exists(os.path.join(q.get_projects_path(), "test1"))
# check the existence of db file
assert os.path.exists(
os.path.join(q.get_projects_path(), "test1", "project.db"))
# make sure project info is in admin.db
admin_interface = factory.get_admindb_interface()
assert admin_interface.check_project_exists("test1")
# clean dummy project
r = self.runner.invoke(delete, ["test1"], input="me\n1234\ny\n")
assert r.exception is None
def swapDB(projectName):
if(projectName == "admin"):
path = conf.settings.DATABASES['admin']['NAME']
else:
path = os.path.join(q.get_projects_path(), projectName, "project.db")
print("old db :", conf.settings.DATABASES['project']['NAME'])
conf.settings.DATABASES['project']['NAME'] = path
print("new db :", conf.settings.DATABASES['project']['NAME'])
def test_bad_input(self):
"""Test exception handling for bad inputs."""
# data file must be jsonl
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.j"],
input="1234\n1234\n")
assert isinstance(r.exception, NotJSONLFileError)
# the following should pass
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert r.exception is None
# collaborator input must be jsonl
r = self.runner.invoke(create, [
"test2", "me", "data/sample_triplets.jsonl", "-a",
"data/sample_collaborators1.j"
],
input="1234\n1234\n")
assert isinstance(r.exception, NotJSONLFileError)
# if a data file doesn't follow quesadiya format, it should give KeyError
r = self.runner.invoke(
create, ["test2", "me", "data/sample_collaborators1.jsonl"],
input="1234\n1234\n")
assert isinstance(r.exception, KeyError)
# the following should pass
r = self.runner.invoke(create, [
"test2", "me", "data/sample_triplets.jsonl", "-a",
"data/sample_collaborators1.jsonl"
],
input="1234\n1234\n")
assert r.exception is None
# if input file doesn't exist, it doesn't create project folder and
# row in admin.db
r = self.runner.invoke(create, [
"test3",
"me",
"data/bluh.jsonl",
],
input="1234\n1234\n")
assert isinstance(r.exception, FileNotFoundError)
assert not os.path.exists(os.path.join(q.get_projects_path(), "test3"))
# `all` is reserved for internal use
r = self.runner.invoke(create,
["all", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert isinstance(r.exception, QuesadiyaCommandError)
# `admin` is reserved for internal use
r = self.runner.invoke(create,
["admin", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert isinstance(r.exception, QuesadiyaCommandError)
# clean dummy projects
r = self.runner.invoke(delete, ["test1"], input="me\n1234\ny\n")
assert r.exception is None
r = self.runner.invoke(delete, ["test2"], input="me\n1234\ny\n")
assert r.exception is None
def test_default_action(self):
"""Test the command deletes a project."""
# create dummy project
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert r.exception is None
# check the existence of project folder
assert os.path.exists(os.path.join(q.get_projects_path(), "test1"))
# check admin.db has the record
admin_interface = factory.get_admindb_interface()
assert admin_interface.check_project_exists("test1")
# delete the project
r = self.runner.invoke(delete, ["test1"], input="me\n1234\ny\n")
assert r.exception is None
# make sure it deletes project folder
assert not os.path.exists(os.path.join(q.get_projects_path(), "test1"))
# make sure record doesn't exist in admin.db
assert not admin_interface.check_project_exists("test1")
def init_admin():
# create `projects` folder
if not os.path.exists(q.get_projects_path()):
try:
os.mkdir(q.get_projects_path())
except PermissionError:
raise PermissionError(
"Permission is denied to create a project folder under {}. "
"Make sure you have the right permission to create folder, or "
"try `pip install . --user`".format(base_dir))
# create admin database file and define schema
db_uri = 'sqlite:///' + os.path.join(q.get_projects_path(), "admin.db")
engine = create_engine(db_uri, echo=False, encoding="utf-8")
# creates admin db tables inside
AdminDB.Base.metadata.create_all(engine)
# import django table from file
# create django tables in admin.db
with engine.connect() as con:
django_database.create_django_tables(con)
def test_default_action(self):
"""Test command exports output file w/ or w/o text."""
# create a dummy project
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert r.exception is None
# insert dummy samples and set status finished for testing export
project_dir = os.path.join(q.get_projects_path(), "test1")
projectdb_interface = factory.get_projectdb_interface(project_dir)
with projectdb_interface.session_context_manager(
expire_on_commit=True) as session:
session.query(TripletDataset)\
.update({"status": TripletStatusEnum.finished})
session.query(TripletDataset)\
.update({"positive_sample_id": self.dummy_positive_sample})
session.query(TripletDataset)\
.update({"negative_sample_id": self.dummy_negative_sample})
# export data
output_path = os.path.join(self.this_dir, "output.jsonl")
assert not os.path.exists(output_path)
r = self.runner.invoke(export, ["test1", output_path],
input="me\n1234\ny\n")
assert r.exception is None
assert os.path.exists(output_path)
# import output and check its data
expected_file_path = os.path.join(self.this_dir,
"expected_output.jsonl")
expected = [x for x in jsonlines.open(expected_file_path)]
output = [x for x in jsonlines.open(output_path)]
for x, y in zip(expected, output):
assert x == y
# remove output file
os.remove(output_path)
output_with_text_path = os.path.join(self.this_dir,
"output_with_text.jsonl")
assert not os.path.exists(output_with_text_path)
# export project with text
r = self.runner.invoke(export, ["test1", output_with_text_path, "-i"],
input="me\n1234\ny\n")
assert r.exception is None
assert os.path.exists(output_with_text_path)
# import output and check its data
expected_file_path = os.path.join(self.this_dir,
"expected_output_with_text.jsonl")
expected_with_text = [x for x in jsonlines.open(expected_file_path)]
output_with_text = [x for x in jsonlines.open(output_with_text_path)]
for x, y in zip(expected_with_text, output_with_text):
assert x == y
# remove file and delete project
os.remove(output_with_text_path)
r = self.runner.invoke(delete, ["test1"], input="me\n1234\ny\n")
assert r.exception is None
def operator(project_name):
admin_interface = factory.get_admindb_interface()
if not admin_interface.check_project_exists(project_name):
raise ProjectNotExistError(project_name)
if admin_interface.is_project_running(project_name):
raise ProjectRunningError(project_name, "`quesadiya delete`")
if not utils.admin_auth(project_name):
raise AuthenticationError(project_name)
conf = click.confirm("Are you sure you want to continue?")
if conf:
click.echo("Successfully deleted project '{}'".format(project_name))
# delete directory first
shutil.rmtree(os.path.join(quesadiya.get_projects_path(),
project_name))
# then, delete row in admin.db
admin_interface.delete_project(project_name)
else:
click.echo("Stop deleting {}".format(project_name))
def get_projectdb_interface(project_name):
project_dir = os.path.join(quesadiya.get_projects_path(), project_name)
engine = _get_projectdb_engine(db_dir_path=project_dir)
interface = SQLAlchemyInterface(engine=engine)
return interface
import os
import quesadiya
# for more configurations:
# https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine
CONFIG_ADMINDB = {
"echo": False,
"encoding": "utf-8",
"pool_pre_ping": True # turn on pessimistic disconnect handling
}
# NOTE: make config customizable in the future
CONFIG_PROJECTDB = {
"echo": False,
"encoding": "utf-8",
"pool_pre_ping": True # turn on pessimistic disconnect handling
}
# path to admin.db
ADMIN_PATH = os.path.join(quesadiya.get_projects_path(), "admin.db")
# project database name
PROJECTDB_NAME = "project.db"
],
},
},
]
WSGI_APPLICATION = 'apps.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.1/ref/settings/#databases
# database_root = os.path.normpath(os.path.abspath(__file__) + os.sep + os.pardir +
# os.sep + os.pardir + os.sep + os.pardir + os.sep + "/projects" + os.sep + "admin.db")
database_sample = os.path.normpath(
os.path.abspath(__file__) + os.sep + os.pardir + os.sep + os.pardir +
os.sep + os.pardir + os.sep + "/projects" + os.sep + "2" + os.sep +
"project.db")
database_root = os.path.join(q.get_projects_path(), "admin.db")
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
# 'NAME': BASE_DIR / 'db.sqlite3',
'NAME': database_root
},
'project': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': database_root
},
'admin': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': database_root
},
}
def create_connection(project_name):
"""Create new database connection."""
db = conf.settings.DATABASES["default"].copy()
db['NAME'] = os.path.join(q.get_projects_path(),
project_name, "project.db")
conf.settings.DATABASES[project_name] = db
def operator(project_name, project_description, admin_name, input_data_path,
admin_contact, collaborator_input_path):
# initial checks
if input_data_path[-6:] != ".jsonl":
raise NotJSONLFileError("`DATAPATH`", input_data_path)
if (collaborator_input_path is not None) and \
(collaborator_input_path[-6:] != ".jsonl"):
raise NotJSONLFileError("`--add-collabortos`", collaborator_input_path)
if project_name == "all":
raise QuesadiyaCommandError("`all` is reserved for use by Quesadiya.")
if project_name == "admin":
raise QuesadiyaCommandError(
"`admin` is reserved for use by Quesadiya.")
utils.check_file_path(input_data_path)
if collaborator_input_path is not None:
utils.check_file_path(collaborator_input_path)
admin_interface = factory.get_admindb_interface()
project_dir = os.path.join(quesadiya.get_projects_path(), project_name)
if admin_interface.check_project_exists(project_name):
raise ProjectExistsError(project_name)
# load data and format it to be inserted into project.db
# if data doesn't follow quesadiya format, it doesn't create a project folder
triplets, candidates, sample_text = \
utils.load_format_dataset(input_path=input_data_path)
# create folder for project inside `projects` dir and insert project.db in it
try:
os.mkdir(project_dir)
except PermissionError:
raise PermissionError(
"Permission is denied to create a project folder under {}. "
"Make sure you have the right permission to create folder under "
"the directory.".format(quesadiya.get_projects_path()))
# ask admin password
admin_password = click.prompt("Admin password",
hide_input=True,
confirmation_prompt=True)
# create project.db
factory.init_projectdb(project_dir)
# get interface
projectdb_interface = factory.get_projectdb_interface(project_name)
# start inserting rows into tables in project.db
click.echo(
"Inserting data. This may take a while ... ".format(project_name))
# showing spinner
with click_spinner.spinner():
projectdb_interface.triplets_bulk_insert(triplets)
projectdb_interface.candidate_groups_bulk_insert(candidates)
projectdb_interface.sample_text_bulk_insert(sample_text)
# insert project into admin.db
admin_interface.insert_project(project_name=project_name,
project_description=project_description,
admin_contact=admin_contact,
status=ProjectStatusEnum.not_running)
# create django tables in project.db and insert values
with projectdb_interface.engine.connect() as con:
create_django_tables(con)
# encode password
insert_admin(con=con,
password=admin_password,
admin_name=admin_name,
date_time=datetime.now())
if collaborator_input_path is not None:
add_collaborators(engine=projectdb_interface.engine,
project_name=project_name,
input_path=collaborator_input_path)
click.echo("done")
# insert bulk data into database
click.echo("Finish creating a new project '{}'".format(project_name))