def batch_load(project,
workspace,
headerline,
entity_data,
chunk_size=500,
model='flexible'):
""" Submit a large number of entity updates in batches of chunk_size """
# Parse the entity type from the first cell, e.g. "entity:sample_id"
# First check that the header is valid
# if not valid_headerline(headerline, model):
# print("Invalid loadfile header:\n" + headerline)
# return 1
update_type = "membership" if headerline.startswith(
"membership") else "entity"
etype = headerline.split('\t')[0].split(':')[1].replace("_id", "")
# Split entity_data into chunks
total = int(len(entity_data) / chunk_size) + 1
batch = 0
for i in range(0, len(entity_data), chunk_size):
batch += 1
print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format(
etype, update_type, i + 1, min(i + chunk_size, len(entity_data)),
batch, total))
this_data = headerline + '\n' + '\n'.join(
entity_data[i:i + chunk_size])
# Now push the entity data to firecloud
r = fapi.upload_entities(project, workspace, this_data, model)
fapi._check_response_code(r, 200)
return 0
def import_entities(self, entities):
"""Upload entity objects.
Args:
entities: iterable of firecloud.Entity objects.
"""
edata = Entity.create_payload(entities)
r = fapi.upload_entities(self.namespace, self.name,
edata, self.api_url)
fapi._check_response_code(r, 201)
def upload_table(namespace, workspace, table, label):
# upload new samples
a = fapi.upload_entities(namespace,
workspace,
entity_data=table.to_csv(index=False, sep="\t"),
model='flexible')
if a.status_code == 200:
print(f'Uploaded {len(table)} {label} rows successfully.')
else:
print(a.json())
def create_set(self, set_id, etype, entities):
"""Create a set of entities and upload to FireCloud.
Args
etype (str): one of {"sample, "pair", "participant"}
entities: iterable of firecloud.Entity objects.
"""
if etype not in {"sample", "pair", "participant"}:
raise ValueError("Unsupported entity type:" + str(etype))
payload = "membership:" + etype + "_set_id\t" + etype + "_id\n"
for e in entities:
if e.etype != etype:
msg = "Entity type '" + e.etype + "' does not match "
msg += "set type '" + etype + "'"
raise ValueError(msg)
payload += set_id + '\t' + e.entity_id + '\n'
r = fapi.upload_entities(self.namespace, self.name,
payload, self.api_url)
fapi._check_response_code(r, 201)