def batch_load(project, workspace, headerline, entity_data, chunk_size=500, model='flexible'): """ Submit a large number of entity updates in batches of chunk_size """ # Parse the entity type from the first cell, e.g. "entity:sample_id" # First check that the header is valid # if not valid_headerline(headerline, model): # print("Invalid loadfile header:\n" + headerline) # return 1 update_type = "membership" if headerline.startswith( "membership") else "entity" etype = headerline.split('\t')[0].split(':')[1].replace("_id", "") # Split entity_data into chunks total = int(len(entity_data) / chunk_size) + 1 batch = 0 for i in range(0, len(entity_data), chunk_size): batch += 1 print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format( etype, update_type, i + 1, min(i + chunk_size, len(entity_data)), batch, total)) this_data = headerline + '\n' + '\n'.join( entity_data[i:i + chunk_size]) # Now push the entity data to firecloud r = fapi.upload_entities(project, workspace, this_data, model) fapi._check_response_code(r, 200) return 0
def import_entities(self, entities): """Upload entity objects. Args: entities: iterable of firecloud.Entity objects. """ edata = Entity.create_payload(entities) r = fapi.upload_entities(self.namespace, self.name, edata, self.api_url) fapi._check_response_code(r, 201)
def upload_table(namespace, workspace, table, label): # upload new samples a = fapi.upload_entities(namespace, workspace, entity_data=table.to_csv(index=False, sep="\t"), model='flexible') if a.status_code == 200: print(f'Uploaded {len(table)} {label} rows successfully.') else: print(a.json())
def create_set(self, set_id, etype, entities): """Create a set of entities and upload to FireCloud. Args etype (str): one of {"sample, "pair", "participant"} entities: iterable of firecloud.Entity objects. """ if etype not in {"sample", "pair", "participant"}: raise ValueError("Unsupported entity type:" + str(etype)) payload = "membership:" + etype + "_set_id\t" + etype + "_id\n" for e in entities: if e.etype != etype: msg = "Entity type '" + e.etype + "' does not match " msg += "set type '" + etype + "'" raise ValueError(msg) payload += set_id + '\t' + e.entity_id + '\n' r = fapi.upload_entities(self.namespace, self.name, payload, self.api_url) fapi._check_response_code(r, 201)