def read_data(
antibodies_tsv_path,
dataset_path,
):
# ab_list = antibodies.read_antibodies(config.labels, antibodies_tsv_path)
ab_table = tables.read_tsv(antibodies_tsv_path)
grid = grids.table_to_grid(config.prefixes, config.fields, ab_table)
cell = grids.value_cell("")
cell["colspan"] = len(grid["headers"][0])
grid["headers"].insert(0, [cell])
for root, dirs, files in os.walk(dataset_path):
for name in files:
if name.startswith("antibodies"):
continue
if name.endswith("-valid-expanded.tsv"):
assays_tsv_path = os.path.join(root, name)
assay_name = name.replace("-submission-valid-expanded.tsv",
"").replace("-", " ")
assay_table = tables.read_tsv(assays_tsv_path)
columns = len(assay_table[0].keys()) - 1
assay_grid = grids.table_to_grid(config.prefixes,
config.fields, assay_table)
ab_map = {}
for row in assay_grid["rows"]:
ab_label = row[0]["value"]
row.pop(0)
ab_map[ab_label] = row
header = grids.value_cell(assay_name)
header["colspan"] = columns
grid["headers"][0].append(header)
grid["headers"][1] += assay_grid["headers"][0][1:]
for row in grid["rows"]:
ab_label = row[0]["value"].replace(":", " ")
if ab_label in ab_map:
row += ab_map[ab_label]
else:
for column in range(0, columns):
row.append(grids.value_cell(""))
grid[
"message"] = "This is the public view with all antibodies (blinded) and assays."
return grid
def fill(rows=[]):
"""Fill the antibodies submission template, returning a list of grids."""
instructions_rows = []
instructions = """CoVIC-DB Antibodies Submission
Version 1.2.3
Add your antibodies to the 'Antibodies' sheet.
Please use consecutive rows (no blank rows).
Do not edit the header row of the 'Antibodies' sheet.
Do not edit the other sheets.
"""
for line in instructions.strip().splitlines():
instructions_rows.append([grids.value_cell(line)])
instructions_rows[0][0]["bold"] = True
instructions_rows[0][0]["width"] = 18
instructions_rows[0].append(grids.value_cell(""))
instructions_rows[0][1]["width"] = 70
instructions_rows.append([])
for header in headers:
description = header["description"].strip().splitlines()
d = description.pop(0)
instructions_rows.append([grids.value_cell(header["label"]), grids.value_cell(d)])
instructions_rows[-1][0]["bold"] = True
for d in description:
instructions_rows.append([grids.value_cell(""), grids.value_cell(d)])
instructions_rows[-1][0]["bold"] = True
terminology_tables = OrderedDict()
for header in headers:
if "terminology" in header:
terminology_tables[header["label"]] = header["terminology"]
terminology_tables_lengths = [len(t) for t in terminology_tables.values()]
terminology_table = []
for i in range(0, max(terminology_tables_lengths)):
newrow = OrderedDict()
for key, values in terminology_tables.items():
newrow[key] = values[i] if i < len(values) else ""
terminology_table.append(newrow)
terminology_grid = grids.table_to_grid({}, {}, terminology_table)
terminology_grid["title"] = "Terminology"
terminology_grid["locked"] = True
return [
{"title": "Instructions", "locked": True, "rows": instructions_rows},
{
"title": "Antibodies",
"active": True,
"activeCell": "A2",
"headers": [headers],
"rows": rows,
},
terminology_grid,
]
def fill(assay_type, rows=[]):
"""Fill the assay submission template, returning a list of grids."""
assay_headers = get_assay_headers(assay_type)
instructions = """CoVIC-DB Dataset Submission
Add your results to the 'Dataset' sheet. Do not edit the other sheets.
Columns:
"""
for header in assay_headers:
example = ""
if "example" in header and header["example"].strip():
example = f" (e.g. {header['example']})"
instructions += f"- {header['label']}: {header['description']}{example}\n"
instructions_rows = []
for line in instructions.strip().splitlines():
instructions_rows.append([grids.value_cell(line)])
instructions_rows[0][0]["bold"] = True
terminology_tables = OrderedDict()
for header in assay_headers:
if "terminology" in header:
terminology_tables[header["label"]] = header["terminology"]
terminology_tables_lengths = [len(t) for t in terminology_tables.values()]
terminology_table = []
if len(terminology_tables) > 0:
for i in range(0, max(terminology_tables_lengths)):
newrow = OrderedDict()
for key, values in terminology_tables.items():
newrow[key] = values[i] if i < len(values) else ""
terminology_table.append(newrow)
terminology_grid = grids.table_to_grid({}, {}, terminology_table)
else:
terminology_grid = {}
terminology_grid["title"] = "Terminology"
terminology_grid["locked"] = True
return [
{
"title": "Instructions",
"locked": True,
"rows": instructions_rows
},
{
"title": "Dataset",
"active": True,
"activeCell": "A2",
"headers": [assay_headers],
"rows": [row[0:len(assay_headers)] for row in rows],
},
terminology_grid,
]
def fill(datatype, source=None):
"""Given a datatype string and an optional table of data,
fill the template for the given datatype,
and return a response with "grids"."""
if source:
response = read(source)
if failed(response):
return response
table = response["table"]
grid = grids.table_to_grid(config.prefixes, config.fields, table)
response = fill_rows(datatype, grid["rows"])
response["table"] = table
response["grid"] = grid
else:
response = fill_rows(datatype)
return response
def to_html(response, prefixes={}, fields={}):
lines = ["<div>"]
if "message" in response:
lines.append(" <p>{0}</p>".format(response["message"]))
if "exception" in response:
lines.append(" <p>{0}</p>".format(str(response["exception"])))
if "errors" in response:
lines.append(" <p>Errors</p>")
lines.append(" <ul>")
for error in response["errors"]:
lines.append(" <li>{0}</li>".format(error))
lines.append(" </ul>")
if "grid" in response:
lines.append(grids.grid_to_html(response["grid"]))
elif "table" in response:
lines.append(grids.grid_to_html(grids.table_to_grid(prefixes, fields, response["table"])))
lines.append("</div>")
return "\n".join(lines)
def convert(source, destination):
"""Given a source and a destimation (format or path)
convert the table to that format
and return a response with a "content" key."""
table = None
grid = None
if grids.is_grid(source):
grid = source
else:
response = read(source)
if failed(response):
return response
table = response["table"]
output_format = destination.lower()
if output_format not in ["tsv", "html"]:
filename, extension = os.path.splitext(destination)
output_format = extension.lower().lstrip(".")
if output_format.lower() == "tsv":
content = tables.table_to_tsv_string(table)
return success({
"table": table,
"content type": responses.tsv,
"content": content
})
elif output_format.lower() == "html":
if not grid:
grid = grids.table_to_grid(config.prefixes, config.fields, table)
html = grids.grid_to_html(grid)
content = templates.render_html("templates/grid.html", {"html": html})
return success({
"table": table,
"grid": grid,
"html": html,
"content type": responses.html,
"content": content,
})
else:
return failure(f"Unsupported output format for '{destination}'")
def test_grid():
grid = "Foo"
assert not grids.is_grid(grid)
grid = {}
assert not grids.is_grid(grid)
grid = {"rows": {}}
assert not grids.is_grid(grid)
grid = {"rows": []}
assert not grids.is_grid(grid)
grid = {"rows": ["Foo"]}
assert not grids.is_grid(grid)
grid = {"rows": [{"value": "foo"}]}
assert not grids.is_grid(grid)
grid = {"headers": "Foo", "rows": ["Foo"]}
assert not grids.is_grid(grid)
prefixes = {"ex": "http://example.com/"}
fields = {"foo_id": {"label": "Foo"}}
labelled_table = [OrderedDict({"foo_id": "ex:bar", "foo_label": "Bar"})]
grid = {
"headers": [[{
"label": "Foo",
"value": "foo_id"
}]],
"rows": [[{
"iri": "http://example.com/bar",
"label": "Bar",
"value": "ex:bar"
}]],
}
assert grids.table_to_grid(prefixes, fields, labelled_table) == grid
def submit(name, email, organization, table):
"""Given a new table of antibodies:
1. validate it
2. assign IDs and append them to the secrets,
3. append the blinded antibodies to the staging table,
4. return a response with merged IDs."""
response = validate(table)
if failed(response):
return response
table = response["table"] # blank rows removed
if not config.secret:
return failure("CVDB_SECRET directory is not configured")
secret = []
path = os.path.join(config.secret.working_tree_dir, "antibodies.tsv")
if os.path.isfile(path):
secret = tables.read_tsv(path)
blind = config.read_blinded_antibodies()
if len(secret) != len(blind):
return failure(f"Different number of antibody rows: {len(secret)} != {len(blind)}")
current_id = "COVIC:0"
if len(blind) > 0:
current_id = blind[-1]["ab_id"]
submission = []
for row in table:
current_id = names.increment_id(current_id)
# secrets: write this to the secret repo
secret_row = OrderedDict()
secret_row["ab_id"] = current_id
secret_row["ab_name"] = row["Antibody name"]
secret_row["ab_details"] = row["Antibody details"]
secret_row["ab_comment"] = row["Antibody comment"]
secret_row["org_name"] = organization
secret_row["submitter_email"] = email
secret.append(secret_row)
# blind: write this to staging/public repos
blind_row = OrderedDict()
blind_row["ab_id"] = current_id
# submission: return this to the submitter
submission_row = OrderedDict()
submission_row["ab_id"] = current_id
submission_row["ab_name"] = row["Antibody name"]
# for each header, add cells to blind and submission
for header in headers[1:]:
column = header["value"]
value = row[header["label"]]
if column.endswith("_label"):
i = config.ids.get(value, "")
blind_row[column.replace("_label", "_id")] = i
submission_row[column.replace("_label", "_id")] = i
submission_row[column] = value
else:
blind_row[column] = value
submission_row[column] = value
blind.append(blind_row)
submission.append(submission_row)
author = Actor(name, email)
# secret
try:
path = os.path.join(config.secret.working_tree_dir, "antibodies.tsv")
tables.write_tsv(secret, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.secret.index.add([path])
config.secret.index.commit("Submit antibodies", author=author, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# staging
try:
path = os.path.join(config.staging.working_tree_dir, "antibodies.tsv")
tables.write_tsv(blind, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.staging.index.add([path])
config.staging.index.commit("Submit antibodies", author=author, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# public
if not config.public:
return failure("CVDB_PUBLIC directory is not configured")
try:
path = os.path.join(config.public.working_tree_dir, "antibodies.tsv")
tables.write_tsv(blind, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.public.index.add([path])
config.public.index.commit("Submit antibodies", author=config.covic, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
grid = grids.table_to_grid(config.prefixes, config.fields, submission)
print("Submitted antibodies")
return success({"table": submission, "grid": grid})
def submit(name, email, dataset_id, table):
"""Given a dataset ID and a new table of assays,
validate it, save it to staging, and commit."""
response = validate(dataset_id, table)
if failed(response):
return response
table = response["table"] # remove blank rows
ab_ids = {}
for ab in config.read_blinded_antibodies():
ab_id = ab["ab_id"]
ab_label = ab_id.replace(":", "-")
ab_ids[ab_label] = ab_id
for row in config.ab_controls.values():
ab_ids[row["label"]] = row["id"]
assay_headers = get_assay_headers(dataset_id)
assays = []
for row in table:
assay = OrderedDict()
for header in assay_headers:
value = header["value"]
label = header["label"]
if value == "ab_label":
row[label] = row[label].strip()
assay["ab_id"] = ab_ids[row[label]]
else:
assay[value] = row[label]
assays.append(assay)
author = Actor(name, email)
# staging
if not config.staging:
return failure("CVDB_STAGING directory is not configured")
dataset_path = os.path.join(config.staging.working_tree_dir, "datasets",
str(dataset_id))
paths = []
try:
set_staging_value(dataset_id, "Dataset status", "submitted")
path = os.path.join(dataset_path, "dataset.yml")
paths.append(path)
except Exception as e:
return failure("Failed to update dataset status", {"exception": e})
try:
path = os.path.join(dataset_path, "assays.tsv")
tables.write_tsv(assays, path)
paths.append(path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.staging.index.add(paths)
config.staging.index.commit(
f"Submit assays to dataset {dataset_id}",
author=author,
committer=config.covic,
)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
grid = grids.table_to_grid(config.prefixes, config.fields, table)
print(f"Submitted assays to dataset {dataset_id}")
return success({"table": table, "grid": grid, "dataset_id": dataset_id})