def upload(workspace: str, graph: str) -> Any:
"""Store a d3 json-encoded graph into the database, with node and edge tables.
`workspace` - the target workspace
`graph` - the target graph
`data` - the json data, passed in the request body. The json data should contain
nodes: [] and links: []
"""
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("graph", graph)
# Get data from the request and load it as json
body = decode_data(request.data)
data = json.load(StringIO(body), object_pairs_hook=OrderedDict)
# Check file structure
errors = validate_d3_json(data)
if len(errors) > 0:
raise ValidationFailed(errors)
node_table_name = f"{graph}_nodes"
edge_table_name = f"{graph}_links"
# Change column names from the d3 format to the arango format
nodes = data["nodes"]
for node in nodes:
node["_key"] = str(node["id"])
del node["id"]
links = data["links"]
for link in links:
link["_from"] = f"{node_table_name}/{link['source']}"
link["_to"] = f"{node_table_name}/{link['target']}"
del link["source"]
del link["target"]
# Create or retrieve the node and edge tables
if loaded_workspace.has_table(node_table_name):
node_table = loaded_workspace.table(node_table_name)
else:
node_table = loaded_workspace.create_table(node_table_name, edge=False)
if loaded_workspace.has_table(edge_table_name):
edge_table = loaded_workspace.table(edge_table_name)
else:
edge_table = loaded_workspace.create_table(edge_table_name, edge=True)
# Insert data
node_table.insert(nodes)
edge_table.insert(links)
loaded_workspace.create_graph(graph, edge_table_name)
return {"nodecount": len(nodes), "edgecount": len(links)}
def upload(workspace: str, graph: str) -> Any:
"""
Store a nested_json tree into the database in coordinated node and edge tables.
`workspace` - the target workspace.
`graph` - the target graph.
`data` - the nested_json data, passed in the request body.
"""
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("graph", graph)
# Set up the parameters.
data = request.data.decode("utf8")
edgetable_name = f"{graph}_edges"
int_nodetable_name = f"{graph}_internal_nodes"
leaf_nodetable_name = f"{graph}_leaf_nodes"
# Set up the database targets.
if loaded_workspace.has_table(edgetable_name):
edgetable = loaded_workspace.table(edgetable_name)
else:
edgetable = loaded_workspace.create_table(edgetable_name, edge=True)
if loaded_workspace.has_table(int_nodetable_name):
int_nodetable = loaded_workspace.table(int_nodetable_name)
else:
int_nodetable = loaded_workspace.create_table(int_nodetable_name,
edge=False)
if loaded_workspace.has_table(leaf_nodetable_name):
leaf_nodetable = loaded_workspace.table(leaf_nodetable_name)
else:
leaf_nodetable = loaded_workspace.create_table(leaf_nodetable_name,
edge=False)
# Analyze the nested_json data into a node and edge table.
(nodes, edges) = analyze_nested_json(data, int_nodetable_name,
leaf_nodetable_name)
# Upload the data to the database.
edgetable.insert(edges)
int_nodetable.insert(nodes[0])
leaf_nodetable.insert(nodes[1])
# Create graph
loaded_workspace.create_graph(graph, edgetable_name)
return {
"edgecount": len(edges),
"int_nodecount": len(nodes[0]),
"leaf_nodecount": len(nodes[1]),
}
def link_generator(loaded_workspace: Workspace,
loaded_graph: Graph) -> Generator[str, None, None]:
"""Generate the JSON list of links."""
# Checks for node tables that have a `_nodes` suffix.
# If matched, removes this suffix.
table_nodes_pattern = re.compile(r"^([^\d_]\w+)_nodes(/.+)")
# Done this way to preserve logic in the future case of multiple edge tables
edge_tables: List[str] = [loaded_graph.edge_table()]
comma = ""
for edge_table in edge_tables:
edges = loaded_workspace.table(edge_table).rows()["rows"]
for edge in edges:
source = edge["_from"]
target = edge["_to"]
source_match = table_nodes_pattern.search(source)
target_match = table_nodes_pattern.search(target)
if source_match and target_match:
source = "".join(source_match.groups())
target = "".join(target_match.groups())
edge["source"] = source
edge["target"] = target
del edge["_from"]
del edge["_to"]
yield f"{comma}{json.dumps(edge, separators=(',', ':'))}"
comma = comma or ","
def download(workspace: str, table: str) -> Any:
"""
Download a table from the database as a CSV file.
`workspace` - the target workspace
`table` - the target table
"""
loaded_workspace = Workspace(workspace)
if not loaded_workspace.has_table(table):
raise NotFound("table", table)
loaded_table = loaded_workspace.table(table)
table_rows = loaded_table.rows()["rows"]
fields = loaded_table.headers()
def csv_row_generator() -> Generator[str, None, None]:
header_line = StringIO()
writer = csv.DictWriter(header_line, fieldnames=fields)
writer.writeheader()
yield header_line.getvalue()
for csv_row in generate_filtered_docs(table_rows):
line = StringIO()
writer = csv.DictWriter(line, fieldnames=fields)
writer.writerow(csv_row)
yield line.getvalue()
response = Response(csv_row_generator(), mimetype="text/csv")
response.headers[
"Content-Disposition"] = f"attachment; filename={table}.csv"
response.headers["Content-type"] = "text/csv"
return response
def node_generator(loaded_workspace: Workspace,
loaded_graph: Graph) -> Generator[str, None, None]:
"""Generate the JSON list of nodes."""
comma = ""
node_tables = loaded_graph.node_tables()
for node_table in node_tables:
table_nodes = loaded_workspace.table(node_table).rows()["rows"]
for node in table_nodes:
node["id"] = node["_key"]
del node["_key"]
yield f"{comma}{json.dumps(node, separators=(',', ':'))}"
comma = comma or ","
def upload(workspace: str, graph: str) -> Any:
"""
Store a newick tree into the database in coordinated node and edge tables.
`workspace` - the target workspace.
`graph` - the target graph.
`data` - the newick data, passed in the request body.
"""
app.logger.info("newick tree")
loaded_workspace = Workspace(workspace)
if loaded_workspace.has_graph(graph):
raise AlreadyExists("graph", graph)
body = decode_data(request.data)
tree = newick.loads(body)
validate_newick(tree)
edgetable_name = f"{graph}_edges"
nodetable_name = f"{graph}_nodes"
if loaded_workspace.has_table(edgetable_name):
edgetable = loaded_workspace.table(edgetable_name)
else:
# Note that edge=True must be set or the _from and _to keys
# will be ignored below.
edgetable = loaded_workspace.create_table(edgetable_name, edge=True)
if loaded_workspace.has_table(nodetable_name):
nodetable = loaded_workspace.table(nodetable_name)
else:
nodetable = loaded_workspace.create_table(nodetable_name, edge=False)
edgecount = 0
nodecount = 0
def read_tree(parent: Optional[str], node: newick.Node) -> None:
nonlocal nodecount
nonlocal edgecount
key = node.name or uuid.uuid4().hex
if not nodetable.row(key):
nodetable.insert([{"_key": key}])
nodecount = nodecount + 1
for desc in node.descendants:
read_tree(key, desc)
if parent:
edgetable.insert(
[
{
"_from": f"{nodetable_name}/{parent}",
"_to": f"{nodetable_name}/{key}",
"length": node.length,
}
]
)
edgecount += 1
read_tree(None, tree[0])
loaded_workspace.create_graph(graph, edgetable_name)
return {"edgecount": edgecount, "nodecount": nodecount}