def read_file(request_files):
"""Given a submitted_id string, a submitter_label string,
and Django request.FILES object with one file,
return a request object with a "filename" string and a "bytes" BytesIO."""
if len(request_files.keys()) < 1:
return failure("No files in request")
if len(request_files.keys()) > 1:
return failure("Multiple upload files not allowed")
upload_file = list(request_files.values())[0]
filename = upload_file.name
if not filename.endswith(".xlsx"):
return failure("Only .xlsx files are supported at this time.")
content = BytesIO()
try:
for chunk in upload_file.chunks():
content.write(chunk)
except Exception as e:
return failure("Invalid upload", {"exception": e})
return success({
"filename": filename,
"content type": xlsx,
"content": content
})
def get_value(scope, dataset, key=None):
"""Given a scope (staging or secret), a dataset ID, and an optional key,
return the value or values in the 'data' key."""
try:
return success({"data": datasets.get_value(scope, dataset, key)})
except Exception as e:
return failure(e)
def expand(source, sheet=None):
"""Given a table, return a response in which "table" is the expanded form."""
response = read(source, sheet)
if failed(response):
return response
table = response["table"]
return success({"table": names.label_table(config.labels, table)})
def read(source, sheet=None):
"""Read a source and return a response with a "table" key."""
if tables.is_table(source):
return success({"table": source})
if responses.is_response(source):
if "table" in source:
return success({"table": source["table"]})
else:
return failure(f"Response does not have 'table': '{source}'")
if isinstance(source, str) or hasattr(source, "read"):
return read_path(source, sheet)
if requests.is_request(source):
response = requests.read_file(source)
if failed(response):
return response
table = workbooks.read(response["content"], sheet)
return success({"table": table})
raise Exception(f"Unknown input '{source}'")
def set_value(scope, dataset, key, value):
"""Given a scope (staging or secret), a dataset ID, a key string, and a simle value,
that can be represented in YAML,
add the key and value to the dataset metadata,
maybe overwriting it."""
try:
datasets.set_value(scope, dataset, key, value)
return success()
except Exception as e:
return failure(e)
def read_path(path, sheet=None):
"""Read a TSV or Excel from a path and return a response with a "table" key."""
table = None
filename, extension = os.path.splitext(path)
extension = extension.lower()
if extension == ".xlsx":
table = workbooks.read(path, sheet)
elif extension == ".tsv":
table = tables.read_tsv(path)
else:
return failure(f"Unsupported input format for '{path}'")
return success({"table": table})
def convert(source, destination):
"""Given a source and a destimation (format or path)
convert the table to that format
and return a response with a "content" key."""
table = None
grid = None
if grids.is_grid(source):
grid = source
else:
response = read(source)
if failed(response):
return response
table = response["table"]
output_format = destination.lower()
if output_format not in ["tsv", "html"]:
filename, extension = os.path.splitext(destination)
output_format = extension.lower().lstrip(".")
if output_format.lower() == "tsv":
content = tables.table_to_tsv_string(table)
return success({
"table": table,
"content type": responses.tsv,
"content": content
})
elif output_format.lower() == "html":
if not grid:
grid = grids.table_to_grid(config.prefixes, config.fields, table)
html = grids.grid_to_html(grid)
content = templates.render_html("templates/grid.html", {"html": html})
return success({
"table": table,
"grid": grid,
"html": html,
"content type": responses.html,
"content": content,
})
else:
return failure(f"Unsupported output format for '{destination}'")
def promote(name, email, dataset_id):
author = Actor(name, email)
# staging
if not config.staging:
return failure("CVDB_STAGING directory is not configured")
staging_dataset_path = os.path.join(config.staging.working_tree_dir,
"datasets", str(dataset_id))
paths = []
try:
set_staging_value(dataset_id, "Dataset status", "promoted")
path = os.path.join(staging_dataset_path, "dataset.yml")
paths.append(path)
except Exception as e:
return failure("Failed to update dataset status", {"exception": e})
try:
config.staging.index.add(paths)
config.staging.index.commit(f"Promote dataset {dataset_id}",
author=author,
committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# public
if not config.public:
return failure("CVDB_PUBLIC directory is not configured")
public_dataset_path = os.path.join(config.public.working_tree_dir,
"datasets", str(dataset_id))
try:
os.makedirs(public_dataset_path)
except Exception as e:
return failure(f"Could not create '{path}'", {"exception": e})
try:
paths = []
for filename in ["dataset.yml", "assays.tsv"]:
src = os.path.join(staging_dataset_path, filename)
dst = os.path.join(public_dataset_path, filename)
shutil.copyfile(src, dst)
paths.append(dst)
except Exception as e:
return failure(f"Could not copy '{src}' to '{dst}'", {"exception": e})
try:
config.public.index.add(paths)
config.public.index.commit(f"Promote dataset {dataset_id}",
author=config.covic,
committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{public_dataset_path}'",
{"exception": e})
print(f"Promoted dataset {dataset_id} from staging to public")
return success({"dataset_id": dataset_id})
def fill_rows(datatype, rows=[]):
"""Given a datatype string and optional rows list,
fill the template for the given datatype,
and return a response with "grids"."""
grids = None
if datatype.lower() == "antibodies":
grids = antibodies.fill(rows)
else:
grids = datasets.fill(datatype, rows)
content = BytesIO()
workbooks.write(grids, content)
return success({
"grids": grids,
"content type": responses.xlsx,
"content": content
})
def validate(headers, table):
"""Given the headers and a (validated!) table,
return a response with "grid" and maybe "errors"."""
errors = []
rows = []
unique = defaultdict(set)
blinded_antibodies = config.read_blinded_antibodies()
ab_ids = [x["ab_id"] for x in blinded_antibodies
] + [x["id"] for x in config.ab_controls.values()]
ab_labels = [x["ab_id"].replace(":", "-")
for x in blinded_antibodies] + list(config.ab_controls.keys())
columns = set()
for header in headers:
try:
columns.add(header["label"])
except KeyError as e:
raise Exception(f"Bad header {header}", e)
new_table = []
for i in range(0, len(table)):
row = table[i]
# Skip blank rows
values = ""
for value in row.values():
values += str(value).strip()
if values == "":
continue
new_table.append(row)
extra_columns = set(row.keys()) - columns
extra_columns.discard(None)
if extra_columns:
extra = ", ".join(extra_columns)
errors.append(f"Extra columns not allowed: {extra}")
missing_columns = columns - set(row.keys())
if missing_columns:
missing = ", ".join(missing_columns)
errors.append(f"Missing columns: {missing}")
newrow = []
for header in headers:
column = header["label"]
error = None
if column not in row:
# Should be handled above
continue
value = str(row[column]).strip()
if "field" in header and header["field"] == "ab_id":
if value not in ab_ids:
error = (
f"'{value}' is not a valid COVIC antibody ID or control antibody ID "
+ "in column 'Antibody ID'")
elif "field" in header and header["field"] == "ab_label":
if value.lower() in ["na", "n/a"]:
continue
if value not in ab_labels:
error = (
f"'{value}' is not a valid COVIC antibody label or control antibody label "
+ "in column 'Antibody label'")
elif "required" in header and header["required"] and value == "":
error = f"Missing required value in column '{column}'"
elif "unique" in header and header["unique"] and value in unique[
column]:
error = f"Duplicate value '{value}' is not allowed in column '{column}'"
elif "terminology" in header and value != "" and value not in header[
"terminology"]:
error = f"'{value}' is not a valid term in column '{column}'"
elif "type" in header and value != "":
error = validate_field(column, header["type"], value)
if "unique" in header and header["unique"]:
unique[column].add(value)
cell = None
if error:
cell = grids.error_cell(value, error)
errors.append("Error in row {0}: {1}".format(i + 2, error))
else:
cell = grids.value_cell(value)
newrow.append(cell)
rows.append(newrow)
table = new_table
grid = {"headers": [headers], "rows": rows}
unique_errors = []
for error in errors:
if error not in unique_errors:
unique_errors.append(error)
errors = unique_errors
error_count = len(errors)
if error_count > 0:
return failure(
f"There were {error_count} errors",
{
"errors": errors,
"table": table,
"grid": grid
},
)
return success({"table": table, "grid": grid})
def submit(name, email, organization, table):
"""Given a new table of antibodies:
1. validate it
2. assign IDs and append them to the secrets,
3. append the blinded antibodies to the staging table,
4. return a response with merged IDs."""
response = validate(table)
if failed(response):
return response
table = response["table"] # blank rows removed
if not config.secret:
return failure("CVDB_SECRET directory is not configured")
secret = []
path = os.path.join(config.secret.working_tree_dir, "antibodies.tsv")
if os.path.isfile(path):
secret = tables.read_tsv(path)
blind = config.read_blinded_antibodies()
if len(secret) != len(blind):
return failure(f"Different number of antibody rows: {len(secret)} != {len(blind)}")
current_id = "COVIC:0"
if len(blind) > 0:
current_id = blind[-1]["ab_id"]
submission = []
for row in table:
current_id = names.increment_id(current_id)
# secrets: write this to the secret repo
secret_row = OrderedDict()
secret_row["ab_id"] = current_id
secret_row["ab_name"] = row["Antibody name"]
secret_row["ab_details"] = row["Antibody details"]
secret_row["ab_comment"] = row["Antibody comment"]
secret_row["org_name"] = organization
secret_row["submitter_email"] = email
secret.append(secret_row)
# blind: write this to staging/public repos
blind_row = OrderedDict()
blind_row["ab_id"] = current_id
# submission: return this to the submitter
submission_row = OrderedDict()
submission_row["ab_id"] = current_id
submission_row["ab_name"] = row["Antibody name"]
# for each header, add cells to blind and submission
for header in headers[1:]:
column = header["value"]
value = row[header["label"]]
if column.endswith("_label"):
i = config.ids.get(value, "")
blind_row[column.replace("_label", "_id")] = i
submission_row[column.replace("_label", "_id")] = i
submission_row[column] = value
else:
blind_row[column] = value
submission_row[column] = value
blind.append(blind_row)
submission.append(submission_row)
author = Actor(name, email)
# secret
try:
path = os.path.join(config.secret.working_tree_dir, "antibodies.tsv")
tables.write_tsv(secret, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.secret.index.add([path])
config.secret.index.commit("Submit antibodies", author=author, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# staging
try:
path = os.path.join(config.staging.working_tree_dir, "antibodies.tsv")
tables.write_tsv(blind, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.staging.index.add([path])
config.staging.index.commit("Submit antibodies", author=author, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# public
if not config.public:
return failure("CVDB_PUBLIC directory is not configured")
try:
path = os.path.join(config.public.working_tree_dir, "antibodies.tsv")
tables.write_tsv(blind, path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.public.index.add([path])
config.public.index.commit("Submit antibodies", author=config.covic, committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
grid = grids.table_to_grid(config.prefixes, config.fields, submission)
print("Submitted antibodies")
return success({"table": submission, "grid": grid})
def submit(name, email, dataset_id, table):
"""Given a dataset ID and a new table of assays,
validate it, save it to staging, and commit."""
response = validate(dataset_id, table)
if failed(response):
return response
table = response["table"] # remove blank rows
ab_ids = {}
for ab in config.read_blinded_antibodies():
ab_id = ab["ab_id"]
ab_label = ab_id.replace(":", "-")
ab_ids[ab_label] = ab_id
for row in config.ab_controls.values():
ab_ids[row["label"]] = row["id"]
assay_headers = get_assay_headers(dataset_id)
assays = []
for row in table:
assay = OrderedDict()
for header in assay_headers:
value = header["value"]
label = header["label"]
if value == "ab_label":
row[label] = row[label].strip()
assay["ab_id"] = ab_ids[row[label]]
else:
assay[value] = row[label]
assays.append(assay)
author = Actor(name, email)
# staging
if not config.staging:
return failure("CVDB_STAGING directory is not configured")
dataset_path = os.path.join(config.staging.working_tree_dir, "datasets",
str(dataset_id))
paths = []
try:
set_staging_value(dataset_id, "Dataset status", "submitted")
path = os.path.join(dataset_path, "dataset.yml")
paths.append(path)
except Exception as e:
return failure("Failed to update dataset status", {"exception": e})
try:
path = os.path.join(dataset_path, "assays.tsv")
tables.write_tsv(assays, path)
paths.append(path)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.staging.index.add(paths)
config.staging.index.commit(
f"Submit assays to dataset {dataset_id}",
author=author,
committer=config.covic,
)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
grid = grids.table_to_grid(config.prefixes, config.fields, table)
print(f"Submitted assays to dataset {dataset_id}")
return success({"table": table, "grid": grid, "dataset_id": dataset_id})
def create(name, email, columns=[]):
if not config.staging:
return failure("CVDB_STAGING directory is not configured")
for column in columns:
if column in config.fields:
continue
if column.startswith("obi_") or column.startswith("ontie_"):
assay_id = column.replace("obi_",
"OBI:").replace("ontie_", "ONTIE:")
root_id = (assay_id.replace("_stddev", "").replace(
"_normalized", "").replace("_qualitative", ""))
if assay_id in config.labels:
continue
if root_id in config.labels:
if column.endswith("_stddev"):
continue
if column.endswith("_normalized"):
continue
if column.endswith("_qualitative"):
continue
return failure(f"Unrecognized column '{column}'")
datasets_path = os.path.join(config.staging.working_tree_dir, "datasets")
current_id = 0
if not os.path.exists(datasets_path):
os.makedirs(datasets_path)
if not os.path.isdir(datasets_path):
return failure(f"'{datasets_path}' is not a directory")
for root, dirs, files in os.walk(datasets_path):
for name in dirs:
if re.match(r"\d+", name):
current_id = max(current_id, int(name))
dataset_id = current_id + 1
author = Actor(name, email)
# secret
try:
path = os.path.join(config.secret.working_tree_dir, "datasets.tsv")
datasets = []
if os.path.isfile(path):
datasets = tables.read_tsv(path)
datasets.append(
OrderedDict({
"ds_id": dataset_id,
"submitter_email": email
}))
tables.write_tsv(datasets, path)
except Exception as e:
return failure(f"Failed to update '{path}'", {"exception": e})
try:
config.secret.index.add([path])
config.secret.index.commit(f"Create dataset {dataset_id}",
author=author,
committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
# staging
try:
dataset_path = os.path.join(datasets_path, str(dataset_id))
os.mkdir(dataset_path)
except Exception as e:
return failure(f"Failed to create '{dataset_path}'", {"exception": e})
try:
dataset = {
"Dataset ID": f"ds:{dataset_id}",
"Dataset status": "configured",
"Columns": columns,
}
path = os.path.join(dataset_path, "dataset.yml")
with open(path, "w") as outfile:
yaml.dump(dataset, outfile, sort_keys=False)
except Exception as e:
return failure(f"Failed to write '{path}'", {"exception": e})
try:
config.staging.index.add([path])
config.staging.index.commit(f"Create dataset {dataset_id}",
author=author,
committer=config.covic)
except Exception as e:
return failure(f"Failed to commit '{path}'", {"exception": e})
print(f"Created dataset {dataset_id}")
return success({"dataset_id": dataset_id})
def initialize():
"""Create the global data repositories."""
config.initialize()
return success()