def admin_auth(project_name):
project_dir = os.path.join(quesadiya.get_projects_path(), project_name)
db_interface = factory.get_projectdb_interface(project_dir)
admin_name, admin_password = ask_admin_info()
auth = db_interface.admin_authentication(admin_name=admin_name,
admin_password=admin_password)
return auth
def _update_collaborators(project_name, input_path):
# load new collaborators
new_cs = utils.load_format_collaborators(input_path=input_path)
# get exisitng collaborators
projectdb_interface = factory.get_projectdb_interface(project_name)
current_cs = projectdb_interface.get_collaborators()
# create a set of collaborator names to check exisitance of collaborator
cs_names = set([x["collaborator_name"] for x in current_cs])
num_added = 0
warned = False
with projectdb_interface.engine.connect() as con:
for cs in new_cs:
if cs["name"] not in cs_names:
insert_collaborator(con=con,
password=cs["password"],
collaborator_name=cs["name"],
date_time=datetime.now(),
contact=cs["contact"])
num_added += 1
elif not warned:
warnings.warn(
"Duplicate collaborator name found during inserting "
"new collabortors. Quesadiya skips duplicate "
"collaborators in the new file.", RuntimeWarning)
warned = True
click.echo("Added {} collaborators to project '{}'".format(
num_added, project_name))
def test_default_action(self):
"""Test command exports output file w/ or w/o text."""
# create a dummy project
r = self.runner.invoke(create,
["test1", "me", "data/sample_triplets.jsonl"],
input="1234\n1234\n")
assert r.exception is None
# insert dummy samples and set status finished for testing export
project_dir = os.path.join(q.get_projects_path(), "test1")
projectdb_interface = factory.get_projectdb_interface(project_dir)
with projectdb_interface.session_context_manager(
expire_on_commit=True) as session:
session.query(TripletDataset)\
.update({"status": TripletStatusEnum.finished})
session.query(TripletDataset)\
.update({"positive_sample_id": self.dummy_positive_sample})
session.query(TripletDataset)\
.update({"negative_sample_id": self.dummy_negative_sample})
# export data
output_path = os.path.join(self.this_dir, "output.jsonl")
assert not os.path.exists(output_path)
r = self.runner.invoke(export, ["test1", output_path],
input="me\n1234\ny\n")
assert r.exception is None
assert os.path.exists(output_path)
# import output and check its data
expected_file_path = os.path.join(self.this_dir,
"expected_output.jsonl")
expected = [x for x in jsonlines.open(expected_file_path)]
output = [x for x in jsonlines.open(output_path)]
for x, y in zip(expected, output):
assert x == y
# remove output file
os.remove(output_path)
output_with_text_path = os.path.join(self.this_dir,
"output_with_text.jsonl")
assert not os.path.exists(output_with_text_path)
# export project with text
r = self.runner.invoke(export, ["test1", output_with_text_path, "-i"],
input="me\n1234\ny\n")
assert r.exception is None
assert os.path.exists(output_with_text_path)
# import output and check its data
expected_file_path = os.path.join(self.this_dir,
"expected_output_with_text.jsonl")
expected_with_text = [x for x in jsonlines.open(expected_file_path)]
output_with_text = [x for x in jsonlines.open(output_with_text_path)]
for x, y in zip(expected_with_text, output_with_text):
assert x == y
# remove file and delete project
os.remove(output_with_text_path)
r = self.runner.invoke(delete, ["test1"], input="me\n1234\ny\n")
assert r.exception is None
def operator(project_name, output_path, include_text):
if output_path[-6:] != ".jsonl":
raise NotJSONLFileError("`OUTPUT_PATH`", output_path)
admin_interface = factory.get_admindb_interface()
if not admin_interface.check_project_exists(project_name):
raise ProjectNotExistError(project_name)
if admin_interface.is_project_running(project_name):
raise ProjectRunningError(project_name, "`quesadiya export`")
if not utils.admin_auth(project_name):
raise AuthenticationError(project_name)
# fetch triplets whose status is finished in a project
click.echo("Exporting data...")
with click_spinner.spinner():
projectdb_interface = factory.get_projectdb_interface(project_name)
triplets = []
# if `include_text` is True, it includes text data in json object
if include_text:
rows = projectdb_interface.get_annotated_triplets_with_text()
for row in rows:
triplet = {
"anchor_sample_id":
row.anchor_sample_id,
"positive_sample_id":
row.positive_sample_id,
"negative_sample_id":
row.negative_sample_id,
# split text into paragraps
"anchor_sample_text":
utils.split_text_into_paragraphs(row.anchor_sample_text),
"positive_sample_text":
utils.split_text_into_paragraphs(row.positive_sample_text),
"negative_sample_text":
utils.split_text_into_paragraphs(row.negative_sample_text)
}
triplets.append(triplet)
else:
rows = projectdb_interface.get_annotated_triplets()
for row in rows:
triplet = {
"anchor_sample_id": row.anchor_sample_id,
"positive_sample_id": row.positive_sample_id,
"negative_sample_id": row.negative_sample_id
}
triplets.append(triplet)
with jsonlines.open(output_path, mode="w") as jsonl:
for t in triplets:
jsonl.write(t)
click.echo("{} samples in project '{}' is exported "
"to '{}'".format(len(triplets), project_name, output_path))
def _show_project(project_name, show_collaborators):
admin_interface = factory.get_admindb_interface()
if not admin_interface.check_project_exists(project_name):
raise ProjectNotExistError(project_name)
p = admin_interface.get_project(project_name)
default_table = _DefaultTable()
default_table.add_row(p)
if show_collaborators:
if not utils.admin_auth(project_name):
raise AuthenticationError(project_name)
projectdb_interface = factory.get_projectdb_interface(project_name)
collaborators = projectdb_interface.get_collaborators()
click.echo(default_table.get_table())
click.echo(_format_collaborators(collaborators))
else:
click.echo(default_table.get_table())
def _transfer_ownership(project_name):
# get new values
new_admin = click.prompt("New admin name")
new_password = click.prompt("New password",
hide_input=True,
confirmation_prompt=True)
# confirm the operation
projectdb_interface = factory.get_projectdb_interface(project_name)
admin = projectdb_interface.get_admin_info()
expected = admin["username"] + "/" + project_name
user_input = click.prompt("Type `{}` to confirm".format(expected))
if user_input != expected:
click.echo(
"Input doesn't match {}. Stop this operation.".format(expected))
else:
projectdb_interface.change_admin_name(new_admin)
# NOTE: Must encrypt password before update
encoded_password = PH.hash(new_password)
projectdb_interface.change_admin_password(encoded_password)
click.echo("The ownership of '{}' is transferred to user: {}".format(
project_name, new_admin))
def operator(project_name, project_description, admin_name, input_data_path,
admin_contact, collaborator_input_path):
# initial checks
if input_data_path[-6:] != ".jsonl":
raise NotJSONLFileError("`DATAPATH`", input_data_path)
if (collaborator_input_path is not None) and \
(collaborator_input_path[-6:] != ".jsonl"):
raise NotJSONLFileError("`--add-collabortos`", collaborator_input_path)
if project_name == "all":
raise QuesadiyaCommandError("`all` is reserved for use by Quesadiya.")
if project_name == "admin":
raise QuesadiyaCommandError(
"`admin` is reserved for use by Quesadiya.")
utils.check_file_path(input_data_path)
if collaborator_input_path is not None:
utils.check_file_path(collaborator_input_path)
admin_interface = factory.get_admindb_interface()
project_dir = os.path.join(quesadiya.get_projects_path(), project_name)
if admin_interface.check_project_exists(project_name):
raise ProjectExistsError(project_name)
# load data and format it to be inserted into project.db
# if data doesn't follow quesadiya format, it doesn't create a project folder
triplets, candidates, sample_text = \
utils.load_format_dataset(input_path=input_data_path)
# create folder for project inside `projects` dir and insert project.db in it
try:
os.mkdir(project_dir)
except PermissionError:
raise PermissionError(
"Permission is denied to create a project folder under {}. "
"Make sure you have the right permission to create folder under "
"the directory.".format(quesadiya.get_projects_path()))
# ask admin password
admin_password = click.prompt("Admin password",
hide_input=True,
confirmation_prompt=True)
# create project.db
factory.init_projectdb(project_dir)
# get interface
projectdb_interface = factory.get_projectdb_interface(project_name)
# start inserting rows into tables in project.db
click.echo(
"Inserting data. This may take a while ... ".format(project_name))
# showing spinner
with click_spinner.spinner():
projectdb_interface.triplets_bulk_insert(triplets)
projectdb_interface.candidate_groups_bulk_insert(candidates)
projectdb_interface.sample_text_bulk_insert(sample_text)
# insert project into admin.db
admin_interface.insert_project(project_name=project_name,
project_description=project_description,
admin_contact=admin_contact,
status=ProjectStatusEnum.not_running)
# create django tables in project.db and insert values
with projectdb_interface.engine.connect() as con:
create_django_tables(con)
# encode password
insert_admin(con=con,
password=admin_password,
admin_name=admin_name,
date_time=datetime.now())
if collaborator_input_path is not None:
add_collaborators(engine=projectdb_interface.engine,
project_name=project_name,
input_path=collaborator_input_path)
click.echo("done")
# insert bulk data into database
click.echo("Finish creating a new project '{}'".format(project_name))