def _create_schema(id_: str, schema_content: SchemaContent, elis: ELISClient,
name: Optional[str]) -> None:
original_schema = get_json(elis.get(f"schemas/{id_}"))
new_schema = elis.create_schema(name or original_schema["name"],
schema_content)
for queue_url in original_schema["queues"]:
if queue_url.startswith(elis.url):
queue_url = queue_url[len(elis.url) + 1:]
elis.patch(queue_url, data={"schema": new_schema["url"]})
def get_data(
ctx: click.Context,
queue_id: int,
files: Tuple[str],
indent: int,
ensure_ascii: bool,
format_: str,
output_file: Optional[IO[str]],
):
annotations_to_export = list()
with ELISClient(context=ctx.obj) as elis:
for file in files:
json_response = elis.upload_document(queue_id, file)
annotation_id = get_id(json_response)
annotations_to_export.append(annotation_id)
elis.poll_annotation(annotation_id, _is_done)
export_data = elis.export_data(queue_id, annotations_to_export,
format_)
if format_ == "json":
output = json.dumps(get_json(export_data),
indent=indent,
ensure_ascii=ensure_ascii)
else:
output = get_text(export_data)
click.echo(output.encode("utf-8"), file=output_file, nl=False)
def change_command(
ctx: click.Context,
id_: int,
name: Optional[str],
schema_content: Optional[List[dict]],
connector_id: Optional[int],
locale: Optional[str],
) -> None:
if not any([name, schema_content, connector_id, locale]):
return
data: Dict[str, Any] = {}
if name is not None:
data["name"] = name
if locale is not None:
data["locale"] = locale
with ELISClient(context=ctx.obj) as elis:
if connector_id is not None:
data["connector"] = get_json(
elis.get(f"connectors/{connector_id}"))["url"]
if schema_content is not None:
name = name or elis.get_queue(id_)["name"]
schema_dict = elis.create_schema(f"{name} schema", schema_content)
data["schema"] = schema_dict["url"]
elis.patch(f"queues/{id_}", data)
def create_command(
ctx: click.Context,
username: str,
password: Optional[str],
queue_ids: Tuple[int],
organization_id: Optional[int],
group: str,
locale: str,
) -> None:
"""
Create user with USERNAME and add him to QUEUES specified by ids.
"""
password = password or generate_secret()
with ELISClient(context=ctx.obj) as elis:
if elis.get_users(username=username):
raise click.ClickException(
f"User with username {username} already exists.")
organization = elis.get_organization(organization_id)
workspaces = elis.get_workspaces(organization=organization["id"],
sideloads=(QUEUES, ))
queues = chain.from_iterable(w[str(QUEUES)] for w in workspaces)
queue_urls = [q["url"] for q in queues if q["id"] in queue_ids]
response = elis.create_user(username, organization["url"], queue_urls,
password, group, locale)
click.echo(f"{response['id']}, {password}")
def create_command(
ctx: click.Context,
name: str,
queue_ids: Tuple[int, ...],
active: bool,
events: Tuple[str, ...],
config_url: str,
config_secret: str,
config_insecure_ssl: bool,
) -> None:
with ELISClient(context=ctx.obj) as elis:
if not queue_ids:
queue_urls = [elis.get_queue()["url"]]
else:
queue_urls = []
for id_ in queue_ids:
queue_dict = elis.get_queue(id_)
if queue_dict:
queue_urls.append(queue_dict["url"])
response = elis.create_webhook(
name=name,
queues=queue_urls,
active=active,
events=list(events),
config_url=config_url,
config_secret=config_secret,
config_insecure_ssl=config_insecure_ssl,
)
click.echo(
f"{response['id']}, {response['name']}, {response['queues']}, {response['events']}, {response['config']['url']}"
)
def list_command(ctx: click.Context, ):
with ELISClient(context=ctx.obj) as elis:
webhooks_list = elis.get_webhooks((QUEUES, ))
headers = [
"id", "name", "events", "queues", "active", "url", "insecure_ssl"
]
def get_row(webhook: dict) -> List[str]:
res = [
webhook["id"],
webhook["name"],
", ".join(e for e in webhook["events"]),
", ".join(str(q.get("id", "")) for q in webhook["queues"]),
webhook["active"],
webhook["config"]["url"],
webhook["config"]["insecure_ssl"],
]
try:
secret_key = webhook["config"]["secret"]
except KeyError:
pass
else:
res.append(secret_key)
if "secret" not in headers:
headers.append("secret")
return res
table = [get_row(webhook) for webhook in webhooks_list]
click.echo(tabulate(table, headers=headers))
def change_command(
ctx: click.Context,
id_: int,
queue_ids: Tuple[int],
group: Optional[str],
locale: Optional[str],
password: Optional[str],
) -> None:
if not any([queue_ids, group, locale, password]):
return
data: Dict[str, Any] = {}
if password is not None:
data["password"] = password
with ELISClient(context=ctx.obj) as elis:
if queue_ids:
data[str(QUEUES)] = [
elis.get_queue(queue)["url"] for queue in queue_ids
]
if group is not None:
data[str(GROUPS)] = [
g["url"] for g in elis.get_groups(group_name=group)
]
if locale is not None:
ui_settings = elis.get_user(id_)["ui_settings"]
data["ui_settings"] = {**ui_settings, "locale": locale}
elis.patch(f"{USERS}/{id_}", data)
def list_command(ctx: click.Context,):
with ELISClient(context=ctx.obj) as elis:
connectors_list = elis.get_connectors((QUEUES,))
headers = ["id", "name", "service url", "queues", "params", "asynchronous"]
def get_row(connector: dict) -> List[str]:
res = [
connector["id"],
connector["name"],
connector["service_url"],
", ".join(str(q.get("id", "")) for q in connector["queues"]),
connector["params"],
connector["asynchronous"],
]
try:
token = connector["authorization_token"]
except KeyError:
pass
else:
res.append(token)
if "authorization_token" not in headers:
headers.append("authorization_token")
return res
table = [get_row(connector) for connector in connectors_list]
click.echo(tabulate(table, headers=headers))
def change_command(
id_: int,
name: Optional[str],
schema_content_file: Optional[IO[bytes]],
connector_id: Optional[int],
locale: Optional[str],
) -> None:
if not any([name, schema_content_file, connector_id, locale]):
return
data: Dict[str, Any] = {}
if name is not None:
data["name"] = name
if locale is not None:
data["locale"] = locale
with ELISClient() as elis:
if connector_id is not None:
data["connector"] = get_json(
elis.get(f"connectors/{connector_id}"))["url"]
if schema_content_file is not None:
name = name or elis.get_queue(id_)["name"]
schema_content = json.load(schema_content_file)
schema_dict = elis.create_schema(f"{name} schema", schema_content)
data["schema"] = schema_dict["url"]
elis.patch(f"queues/{id_}", data)
def list_command(ctx: click.Context, user_ids: Tuple[int],
queue_ids: Tuple[int]) -> None:
"""List all users and their assignments to queues."""
with ELISClient(context=ctx.obj) as elis:
queue_users = elis.get_queues((USERS, ), users=user_ids)
user_queues: Dict[int, List[List[Optional[str]]]] = {}
for queue in queue_users:
if queue_ids and int(queue["id"]) not in queue_ids:
continue
for user in queue["users"]:
user_id = int(user["id"])
if user_ids and user_id not in user_ids:
continue
if user_id not in user_queues:
user_queues[user_id] = [[
user["id"], user["username"], queue["id"], queue["name"]
]]
else:
user_queues[user_id].append(
[None, None, queue["id"], queue["name"]])
user_queues = dict(sorted(user_queues.items()))
click.echo(
tabulate(
chain.from_iterable(user_queues.values()),
headers=["id", "username", "queue id", "queue name"],
))
def change_command(
ctx: click.Context,
id_: str,
queue_ids: Tuple[int],
name: Optional[str],
service_url: str,
auth_token: str,
params: Optional[str],
asynchronous: Optional[bool],
) -> None:
if not any([queue_ids, service_url, auth_token, params, asynchronous]):
return
data: Dict[str, Any] = {}
with ELISClient(context=ctx.obj) as elis:
if queue_ids:
data["queues"] = [elis.get_queue(queue)["url"] for queue in queue_ids]
if name is not None:
data["name"] = name
if service_url is not None:
data["service_url"] = service_url
if auth_token is not None:
data["authorization_token"] = auth_token
if params is not None:
data["params"] = params
if asynchronous is not None:
data["asynchronous"] = asynchronous
elis.patch(f"connectors/{id_}", data)
def create_command(
ctx: click.Context,
name: str,
queue_ids: Tuple[int, ...],
service_url: str,
auth_token: str,
params: Optional[str],
asynchronous: Optional[bool],
) -> None:
token = auth_token or _generate_token()
with ELISClient(context=ctx.obj) as elis:
if not queue_ids:
queue_urls = [elis.get_queue()["url"]]
else:
queue_urls = []
for id_ in queue_ids:
queue_dict = elis.get_queue(id_)
if queue_dict:
queue_urls.append(queue_dict["url"])
response = elis.create_connector(
name=name,
queues=queue_urls,
service_url=service_url,
authorization_token=token,
params=params,
asynchronous=asynchronous,
)
click.echo(f"{response['id']}, {response['name']}, {response['queues']}")
def list_command(ctx: click.Context,) -> None:
with ELISClient(context=ctx.obj) as elis:
queues = elis.get_queues((WORKSPACES, INBOXES, SCHEMAS, USERS, WEBHOOKS))
table = [
[
queue["id"],
queue["name"],
str(queue["workspace"].get("id", "")),
queue["inbox"].get("email", ""),
str(queue["schema"].get("id", "")),
", ".join(str(q.get("id", "")) for q in queue["users"]),
queue["connector"],
", ".join(str(q.get("id", "")) for q in queue["webhooks"]),
]
for queue in queues
]
click.echo(
tabulate(
table,
headers=[
"id",
"name",
"workspace",
"inbox",
"schema",
"users",
"connector",
"webhooks",
],
)
)
def create_command(
name: str,
schema_content_file: IO[bytes],
email_prefix: Optional[str],
bounce_email: Optional[str],
workspace_id: Optional[int],
connector_id: Optional[int],
locale: Optional[str],
) -> None:
schema_content = json.load(schema_content_file)
if email_prefix is not None and bounce_email is None:
raise click.ClickException(
"Inbox cannot be created without specified bounce email.")
with ELISClient() as elis:
workspace_url = elis.get_workspace(workspace_id)["url"]
connector_url = (get_json(
elis.get(f"connectors/{connector_id}"))["url"]
if connector_id is not None else None)
schema_dict = elis.create_schema(f"{name} schema", schema_content)
queue_dict = elis.create_queue(name, workspace_url, schema_dict["url"],
connector_url, locale)
inbox_dict = {"email": "no email-prefix specified"}
if email_prefix is not None:
inbox_dict = elis.create_inbox(f"{name} inbox", email_prefix,
bounce_email, queue_dict["url"])
click.echo(f"{queue_dict['id']}, {inbox_dict['email']}")
def upload_command(ctx: click.Context, id_: str, schema_content: List[dict],
rewrite: bool, name: Optional[str]):
"""
Update schema in ELIS.
"""
upload_func = _rewrite_schema if rewrite else _create_schema
with ELISClient(context=ctx.obj) as elis:
upload_func(id_, schema_content, elis, name)
def add_command(ctx: click.Context, user_ids: Tuple[int],
queue_ids: Tuple[int]) -> None:
with ELISClient(context=ctx.obj) as elis:
for user_id in user_ids:
user = elis.get_user(user_id)
new_queues = user["queues"] + [
elis.get_queue(q_id)["url"] for q_id in queue_ids
]
elis.patch(f"{USERS}/{user_id}", {str(QUEUES): new_queues})
def remove_command(ctx: click.Context, user_ids: Tuple[int],
queue_ids: Tuple[int]) -> None:
with ELISClient(context=ctx.obj) as elis:
for user_id in user_ids:
queues = elis.get_queues(users=[user_id])
new_queues = [
q["url"] for q in queues if int(q["id"]) not in queue_ids
]
elis.patch(f"{USERS}/{user_id}", {str(QUEUES): new_queues})
def list_command(ctx: click.Context, ):
with ELISClient(context=ctx.obj) as elis:
schemas = elis.get_schemas((QUEUES, ))
table = [[
schema["id"], schema["name"],
", ".join(str(s.get("id", "")) for s in schema["queues"])
] for schema in schemas]
click.echo(tabulate(table, headers=["id", "name", "queues"]))
def change_command(id_: str, name: Optional[str]) -> None:
if not any([name]):
return
data: Dict[str, Any] = {}
if name is not None:
data["name"] = name
with ELISClient() as elis:
elis.patch(f"workspaces/{id_}", data)
def create_command(name: str, organization_id: Optional[int]) -> None:
with ELISClient() as elis:
organization_url = elis.get_organization(organization_id)["url"]
res = elis.post("workspaces", {
"name": name,
"organization": organization_url
})
workspace_dict = get_json(res)
click.echo(workspace_dict["id"])
def _create_inbox(
elis: ELISClient,
queue_dict: dict,
email_prefix: Optional[str],
bounce_email: Optional[str],
name: Optional[str],
) -> dict:
return elis.create_inbox(
f"{name or queue_dict['name']} inbox", email_prefix, bounce_email, queue_dict["url"]
)
def list_command():
with ELISClient() as elis:
workspaces = elis.get_workspaces((QUEUES, ))
table = [[
workspace["id"],
workspace["name"],
", ".join(str(q.get("id", "")) for q in workspace["queues"]),
] for workspace in workspaces]
click.echo(tabulate(table, headers=["id", "name", "queues"]))
def list_command(ctx: click.Context, ):
with ELISClient(context=ctx.obj) as elis:
users_list = elis.get_users((QUEUES, GROUPS), is_active=True)
table = [[
user["id"],
user["username"],
", ".join(str(g["name"]) for g in user[str(GROUPS)]),
", ".join(str(q["id"]) for q in user[str(QUEUES)]),
] for user in users_list]
click.echo(tabulate(table, headers=["id", "username", "groups", "queues"]))
def change_command(
ctx: click.Context,
id_: int,
name: Optional[str],
schema_content: Optional[List[dict]],
email_prefix: Optional[str],
bounce_email: Optional[str],
connector_id: Optional[int],
webhook_id: Optional[Tuple[int, ...]],
locale: Optional[str],
) -> None:
if not any(
[name, schema_content, email_prefix, bounce_email, connector_id, locale, webhook_id]
):
return
data: Dict[str, Any] = {}
if name is not None:
data["name"] = name
if locale is not None:
data["locale"] = locale
with ELISClient(context=ctx.obj) as elis:
if email_prefix or bounce_email:
queue_dict = elis.get_queue(id_)
if not queue_dict["inbox"]:
inbox_dict = _create_inbox(elis, queue_dict, email_prefix, bounce_email, name)
else:
inbox_dict = _patch_inbox(elis, queue_dict, email_prefix, bounce_email)
click.echo(
f"{inbox_dict['id']}, {inbox_dict['email']}, {inbox_dict['bounce_email_to']}"
)
if connector_id is not None:
data["connector"] = get_json(elis.get(f"connectors/{connector_id}"))["url"]
if webhook_id:
webhooks_urls = []
for webhook in webhook_id:
webhook_url = get_json(elis.get(f"webhooks/{webhook}"))["url"]
webhooks_urls.append(webhook_url)
data["webhooks"] = webhooks_urls
if schema_content is not None:
name = name or elis.get_queue(id_)["name"]
schema_dict = elis.create_schema(f"{name} schema", schema_content)
data["schema"] = schema_dict["url"]
if data:
elis.patch(f"queues/{id_}", data)
def _patch_inbox(
elis: ELISClient, queue_dict: dict, email_prefix: Optional[str], bounce_email: Optional[str]
) -> dict:
inbox_data: Dict[str, Any] = {}
if email_prefix:
inbox_data["email_prefix"] = email_prefix
if bounce_email:
inbox_data["bounce_email_to"] = bounce_email
inbox_data["bounce_unprocessable_attachments"] = True
_, inbox_id = queue_dict["inbox"].rsplit("/", 1)
return get_json(elis.patch(f"inboxes/{inbox_id}", inbox_data))
def list_command() -> None:
with ELISClient() as elis:
queues = elis.get_queues((WORKSPACES, INBOXES, SCHEMAS, USERS))
table = [[
queue["id"],
queue["name"],
str(queue["workspace"].get("id", "")),
queue["inbox"].get("email", ""),
str(queue["schema"].get("id", "")),
", ".join(str(q.get("id", "")) for q in queue["users"]),
] for queue in queues]
click.echo(
tabulate(
table,
headers=["id", "name", "workspace", "inbox", "schema", "users"]))
class TestELISClient:
api_client = ELISClient(None)
@pytest.mark.usefixtures("mock_login_request")
def test_get_organization_old_api(self, requests_mock, isolated_cli_runner):
organization_json = {"test": "test"}
user_url = f"{USERS_URL}/1"
organization_url = f"{ORGANIZATIONS_URL}/1"
requests_mock.get(f"{API_URL}/v1/auth/user", json={"url": user_url})
requests_mock.get(user_url, json={"organization": organization_url})
requests_mock.get(organization_url, json=organization_json)
with isolated_cli_runner.isolation():
assert organization_json == self.api_client.get_organization()
assert requests_mock.called
@pytest.mark.usefixtures("mock_login_request")
def test_upload_overwrite_filename(self, requests_mock, isolated_cli_runner, mock_file):
original_filename = "empty_file.pdf"
overwritten_filename = "Overwritten filename.pdf"
api_response = {"results": [{"document": DOCUMENT_URL}]}
requests_mock.post(
UPLOAD_ENDPOINT,
additional_matcher=partial(match_uploaded_data, original_filename),
request_headers={"Authorization": f"Token {TOKEN}"},
json={"results": [{"document": DOCUMENT_URL}]},
status_code=201,
)
with isolated_cli_runner.isolation():
assert api_response == self.api_client.upload_document(QUEUE_ID, mock_file)
requests_mock.post(
UPLOAD_ENDPOINT,
additional_matcher=partial(match_uploaded_data, overwritten_filename),
request_headers={"Authorization": f"Token {TOKEN}"},
json={"results": [{"document": DOCUMENT_URL}]},
status_code=201,
)
with isolated_cli_runner.isolation():
assert api_response == self.api_client.upload_document(
QUEUE_ID, mock_file, overwritten_filename
)
class TestELISClient:
api_client = ELISClient()
@pytest.mark.usefixtures("mock_login_request")
def test_get_organization_old_api(self, requests_mock,
isolated_cli_runner):
organization_json = {"test": "test"}
user_url = f"{USERS_URL}/1"
organization_url = f"{ORGANIZATIONS_URL}/1"
requests_mock.get(f"{API_URL}/v1/auth/user", json={"url": user_url})
requests_mock.get(user_url, json={"organization": organization_url})
requests_mock.get(organization_url, json=organization_json)
with isolated_cli_runner.isolation():
assert organization_json == self.api_client.get_organization()
assert requests_mock.called
def delete_command(id_: int) -> None:
with ELISClient() as elis:
workspace = elis.get_workspace(id_)
queues = elis.get_queues(workspace=workspace["id"])
documents = {}
for queue in queues:
res, _ = elis.get_paginated(
"annotations",
{
"page_size": 50,
"queue": queue["id"],
"sideload": "documents"
},
key="documents",
)
documents.update({d["id"]: d["url"] for d in res})
elis.delete({workspace["id"]: workspace["url"], **documents})
def upload_command(id_: str, file: IO[bytes], format_: str, rewrite: bool,
name: Optional[str]):
"""
Update schema in ELIS.
"""
if format_ == "json":
load_func: LoadFunction = json.load
else:
load_func = XlsxToSchema().convert
try:
schema = load_func(file)
except Exception as e:
raise click.ClickException(
f"File {file} could not be loaded. Because of {e}") from e
upload_func = _rewrite_schema if rewrite else _create_schema
with ELISClient() as elis:
upload_func(id_, schema, elis, name)