def download(workspace: str, table: str) -> Any:
"""
Download a table from the database as a CSV file.
`workspace` - the target workspace
`table` - the target table
"""
loaded_workspace = Workspace(workspace)
if not loaded_workspace.has_table(table):
raise NotFound("table", table)
loaded_table = loaded_workspace.table(table)
table_rows = loaded_table.rows()["rows"]
fields = loaded_table.headers()
def csv_row_generator() -> Generator[str, None, None]:
header_line = StringIO()
writer = csv.DictWriter(header_line, fieldnames=fields)
writer.writeheader()
yield header_line.getvalue()
for csv_row in generate_filtered_docs(table_rows):
line = StringIO()
writer = csv.DictWriter(line, fieldnames=fields)
writer.writerow(csv_row)
yield line.getvalue()
response = Response(csv_row_generator(), mimetype="text/csv")
response.headers[
"Content-Disposition"] = f"attachment; filename={table}.csv"
response.headers["Content-type"] = "text/csv"
return response
def download(workspace: str, graph: str) -> Any:
"""Return a graph as a d3 json-encoded graph.
`workspace` - the target workspace
`graph` - the target graph
"""
loaded_workspace = Workspace(workspace)
if not loaded_workspace.has_graph(graph):
raise NetworkNotFound(workspace, graph)
loaded_graph = loaded_workspace.graph(graph)
def d3_json_generator() -> Generator[str, None, None]:
yield """{"nodes":["""
yield from node_generator(loaded_workspace, loaded_graph)
yield """],"links":["""
yield from link_generator(loaded_workspace, loaded_graph)
yield "]}"
response = Response(d3_json_generator(), mimetype="application/json")
response.headers[
"Content-Disposition"] = f"attachment; filename={graph}.json"
response.headers["Content-type"] = "application/json"
return response
def managed_workspace(generated_workspace: Workspace):
"""
Create a workspace, and yield the name of the workspace.
On teardown, deletes the workspace.
"""
yield generated_workspace
generated_workspace.delete()
def create_workspace(workspace: str) -> Any:
"""Create a new workspace."""
# The `require_login()` decorator ensures that a user is logged in
user = current_user()
assert user is not None
Workspace.create(workspace, user)
return workspace
def create_graph(workspace: str,
graph: str,
edge_table: Optional[str] = None) -> Any:
"""Create a graph."""
if not edge_table:
raise RequiredParamsMissing(["edge_table"])
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("Graph", graph)
Workspace(workspace).create_graph(graph, edge_table)
return graph
def upload(workspace: str,
table: str,
key: str = "_key",
overwrite: bool = False) -> Any:
"""
Store a CSV file into the database as a node or edge table.
`workspace` - the target workspace
`table` - the target table
`data` - the CSV data, passed in the request body. If the CSV data contains
`_from` and `_to` fields, it will be treated as an edge table.
"""
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_table(table):
raise AlreadyExists("table", table)
app.logger.info("Bulk Loading")
# Read the request body into CSV format
body = decode_data(request.data)
try:
# Type to a Dict rather than an OrderedDict
rows: List[Dict[str, str]] = list(csv.DictReader(StringIO(body)))
except csv.Error:
raise CSVReadError()
# Perform validation.
validate_csv(rows, key, overwrite)
# Once we reach here, we know that the specified key field must be present,
# and either:
# key == "_key" # noqa: E800
# or key != "_key" and the "_key" field is not present
# or key != "_key" and "_key" is present, but overwrite = True
if key != "_key":
rows = set_table_key(rows, key)
# Check if it's an edge table or not
fieldnames = rows[0].keys()
edges = "_from" in fieldnames and "_to" in fieldnames
# Create table and insert the data
loaded_table = loaded_workspace.create_table(table, edges)
results = loaded_table.insert(rows)
return {"count": len(results)}
def wrapper(workspace: str, *args: Any, **kwargs: Any) -> Any:
user = current_user()
if not is_writer(user, Workspace(workspace)):
raise Unauthorized(
f"You must be a writer of workspace '{workspace}'")
return f(workspace, *args, **kwargs)
def link_generator(loaded_workspace: Workspace,
loaded_graph: Graph) -> Generator[str, None, None]:
"""Generate the JSON list of links."""
# Checks for node tables that have a `_nodes` suffix.
# If matched, removes this suffix.
table_nodes_pattern = re.compile(r"^([^\d_]\w+)_nodes(/.+)")
# Done this way to preserve logic in the future case of multiple edge tables
edge_tables: List[str] = [loaded_graph.edge_table()]
comma = ""
for edge_table in edge_tables:
edges = loaded_workspace.table(edge_table).rows()["rows"]
for edge in edges:
source = edge["_from"]
target = edge["_to"]
source_match = table_nodes_pattern.search(source)
target_match = table_nodes_pattern.search(target)
if source_match and target_match:
source = "".join(source_match.groups())
target = "".join(target_match.groups())
edge["source"] = source
edge["target"] = target
del edge["_from"]
del edge["_to"]
yield f"{comma}{json.dumps(edge, separators=(',', ':'))}"
comma = comma or ","
def aql(workspace: str) -> Any:
"""Perform an AQL query in the given workspace."""
query = request.data.decode("utf8")
if not query:
raise MalformedRequestBody(query)
result = Workspace(workspace).run_query(query)
return util.stream(result)
def test_workspace_create(managed_user):
"""Test that creating a workspace doesn't result in invalid caching."""
workspace_name = uuid4().hex
pre_create = workspace_mapping(workspace_name)
workspace = Workspace.create(workspace_name, managed_user)
post_create = workspace_mapping(workspace_name)
post_create_exists = Workspace.exists(workspace_name)
# Teardown
workspace.delete()
# Asserts
assert pre_create is None
assert post_create is not None
assert post_create_exists
def get_workspaces() -> Any:
"""Return the list of available workspaces, based on the logged in user."""
user = current_user()
# If the user is logged in, return all workspaces visible to them
if user is not None:
return util.stream(user.available_workspaces())
# Otherwise, return only public workspaces
return util.stream(Workspace.list_public())
def set_workspace_permissions(workspace: str) -> Any:
"""Set the permissions on a workspace."""
if set(request.json.keys()) != {
"owner",
"maintainers",
"writers",
"readers",
"public",
}:
raise MalformedRequestBody(request.json)
perms = util.contract_user_permissions(request.json)
return Workspace(workspace).set_permissions(perms).__dict__
def test_workspace_delete(generated_workspace):
"""Tests that deleting a workspace doesn't result in invalid caching."""
pre_delete = workspace_mapping(generated_workspace.name)
generated_workspace.delete()
post_delete = workspace_mapping(generated_workspace.name)
exists_post_delete = Workspace.exists(generated_workspace.name)
# Asserts
assert pre_delete is not None
assert post_delete is None
assert not exists_post_delete
def test_workspace_rename(generated_workspace):
"""Test that renaming a workspace doesn't result in invalid caching."""
new_workspace_name = uuid4().hex
old_workspace_name = generated_workspace.name
pre_rename = workspace_mapping(old_workspace_name)
generated_workspace.rename(new_workspace_name)
post_rename_old = workspace_mapping(old_workspace_name)
post_rename_new = workspace_mapping(new_workspace_name)
new_exists = Workspace.exists(new_workspace_name)
old_exists = Workspace.exists(old_workspace_name)
# Teardown
generated_workspace.delete()
# Asserts
assert pre_rename is not None
assert post_rename_old is None
assert post_rename_new is not None
assert new_exists
assert not old_exists
def node_generator(loaded_workspace: Workspace,
loaded_graph: Graph) -> Generator[str, None, None]:
"""Generate the JSON list of nodes."""
comma = ""
node_tables = loaded_graph.node_tables()
for node_table in node_tables:
table_nodes = loaded_workspace.table(node_table).rows()["rows"]
for node in table_nodes:
node["id"] = node["_key"]
del node["_key"]
yield f"{comma}{json.dumps(node, separators=(',', ':'))}"
comma = comma or ","
def get_node_edges(
workspace: str,
graph: str,
table: str,
node: str,
direction: EdgeDirection = "all",
offset: int = 0,
limit: int = 30,
) -> Any:
"""Return the edges connected to a node."""
allowed = ["incoming", "outgoing", "all"]
if direction not in allowed:
raise BadQueryArgument("direction", direction, allowed)
return (Workspace(workspace).graph(graph).node_edges(
table, node, direction, offset, limit))
def upload(workspace: str, graph: str) -> Any:
"""
Store a newick tree into the database in coordinated node and edge tables.
`workspace` - the target workspace.
`graph` - the target graph.
`data` - the newick data, passed in the request body.
"""
app.logger.info("newick tree")
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("graph", graph)
body = decode_data(request.data)
tree = newick.loads(body)
validate_newick(tree)
edgetable_name = f"{graph}_edges"
nodetable_name = f"{graph}_nodes"
if loaded_workspace.has_table(edgetable_name):
edgetable = loaded_workspace.table(edgetable_name)
else:
# Note that edge=True must be set or the _from and _to keys
# will be ignored below.
edgetable = loaded_workspace.create_table(edgetable_name, edge=True)
if loaded_workspace.has_table(nodetable_name):
nodetable = loaded_workspace.table(nodetable_name)
else:
nodetable = loaded_workspace.create_table(nodetable_name, edge=False)
edgecount = 0
nodecount = 0
def read_tree(parent: Optional[str], node: newick.Node) -> None:
nonlocal nodecount
nonlocal edgecount
key = node.name or uuid.uuid4().hex
if not nodetable.row(key):
nodetable.insert([{"_key": key}])
nodecount = nodecount + 1
for desc in node.descendants:
read_tree(key, desc)
if parent:
edgetable.insert(
[
{
"_from": f"{nodetable_name}/{parent}",
"_to": f"{nodetable_name}/{key}",
"length": node.length,
}
]
)
edgecount += 1
read_tree(None, tree[0])
loaded_workspace.create_graph(graph, edgetable_name)
return {"edgecount": edgecount, "nodecount": nodecount}
def get_table_rows(workspace: str,
table: str,
offset: int = 0,
limit: int = 30) -> Any:
"""Retrieve the rows and headers of a table."""
return Workspace(workspace).table(table).rows(offset, limit)
def create_aql_table(workspace: str, table: str) -> Any:
"""Create a table from an AQL query."""
aql = request.data.decode()
Workspace(workspace).create_aql_table(table, aql)
return table
def get_workspace_tables(workspace: str,
type: TableType = "all") -> Any: # noqa: A002
"""Retrieve the tables of a single workspace."""
tables = Workspace(workspace).tables(type)
return util.stream(tables)
def get_graph_nodes(workspace: str,
graph: str,
offset: int = 0,
limit: int = 30) -> Any:
"""Retrieve the nodes of a graph."""
return Workspace(workspace).graph(graph).nodes(offset, limit)
def get_workspace_permissions(workspace: str) -> Any:
"""Retrieve the permissions of a workspace."""
perms = Workspace(workspace).permissions
return util.expand_user_permissions(perms)
def get_node_data(workspace: str, graph: str, table: str, node: str) -> Any:
"""Return the attributes associated with a node."""
return Workspace(workspace).graph(graph).node_attributes(table, node)
def get_workspace_graph(workspace: str, graph: str) -> Any:
"""Retrieve information about a graph."""
node_tables = Workspace(workspace).graph(graph).node_tables()
edge_table = Workspace(workspace).graph(graph).edge_table()
return {"edgeTable": edge_table, "nodeTables": node_tables}
def upload(workspace: str, graph: str) -> Any:
"""
Store a nested_json tree into the database in coordinated node and edge tables.
`workspace` - the target workspace.
`graph` - the target graph.
`data` - the nested_json data, passed in the request body.
"""
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("graph", graph)
# Set up the parameters.
data = request.data.decode("utf8")
edgetable_name = f"{graph}_edges"
int_nodetable_name = f"{graph}_internal_nodes"
leaf_nodetable_name = f"{graph}_leaf_nodes"
# Set up the database targets.
if loaded_workspace.has_table(edgetable_name):
edgetable = loaded_workspace.table(edgetable_name)
else:
edgetable = loaded_workspace.create_table(edgetable_name, edge=True)
if loaded_workspace.has_table(int_nodetable_name):
int_nodetable = loaded_workspace.table(int_nodetable_name)
else:
int_nodetable = loaded_workspace.create_table(int_nodetable_name,
edge=False)
if loaded_workspace.has_table(leaf_nodetable_name):
leaf_nodetable = loaded_workspace.table(leaf_nodetable_name)
else:
leaf_nodetable = loaded_workspace.create_table(leaf_nodetable_name,
edge=False)
# Analyze the nested_json data into a node and edge table.
(nodes, edges) = analyze_nested_json(data, int_nodetable_name,
leaf_nodetable_name)
# Upload the data to the database.
edgetable.insert(edges)
int_nodetable.insert(nodes[0])
leaf_nodetable.insert(nodes[1])
# Create graph
loaded_workspace.create_graph(graph, edgetable_name)
return {
"edgecount": len(edges),
"int_nodecount": len(nodes[0]),
"leaf_nodecount": len(nodes[1]),
}
def delete_workspace(workspace: str) -> Any:
"""Delete a workspace."""
Workspace(workspace).delete()
return workspace
def delete_graph(workspace: str, graph: str) -> Any:
"""Delete a graph."""
Workspace(workspace).delete_graph(graph)
return graph
def delete_table(workspace: str, table: str) -> Any:
"""Delete a table."""
Workspace(workspace).delete_table(table)
return table
def rename_workspace(workspace: str, name: str) -> Any:
"""Delete a workspace."""
Workspace(workspace).rename(name)
return name
def get_workspace_graphs(workspace: str) -> Any:
"""Retrieve the graphs of a single workspace."""
return util.stream((g["name"] for g in Workspace(workspace).graphs()))