def create_fastq_metadata_yaml(library_id, storage_name, dry_run=False):
"""
Create a metadata.yaml file for a all FQ datasets for a library id.
"""
tantalus_api = TantalusApi()
storage = tantalus_api.get_storage(storage_name)
client = tantalus_api.get_storage_client(storage_name)
for dataset_info, metadata in create_lane_fastq_metadata(
tantalus_api, library_id):
metadata_filename = os.path.join(dataset_info['base_dir'],
'metadata.yaml')
metadata_filepath = tantalus_api.get_filepath(storage_name,
metadata_filename)
metadata_io = io.BytesIO()
metadata_io.write(
yaml.dump(metadata, default_flow_style=False).encode())
logging.info(f'writing metadata to file {metadata_filepath}')
client.write_data(metadata_filename, metadata_io)
logging.info(f'adding {metadata_filepath} to tantalus')
if not dry_run:
file_resource, file_instance = tantalus_api.add_file(
storage_name, metadata_filepath, update=True)
for dataset_id in dataset_info['dataset_ids']:
dataset = tantalus_api.get('sequencedataset', id=dataset_id)
new_file_resources = set(dataset['file_resources'])
new_file_resources.add(file_resource['id'])
tantalus_api.update('sequencedataset',
id=dataset_id,
file_resources=list(new_file_resources))
def add_fastq_metadata_yaml(dataset_id, storage_name, dry_run=False):
"""
Create a metadata.yaml file for a dataset and add to tantalus.
"""
tantalus_api = TantalusApi()
storage = tantalus_api.get_storage(storage_name)
client = tantalus_api.get_storage_client(storage_name)
metadata, base_dir = create_lane_fastq_metadata(tantalus_api, dataset_id)
metadata_filename = os.path.join(base_dir, 'metadata.yaml')
metadata_filepath = tantalus_api.get_filepath(storage_name,
metadata_filename)
metadata_io = io.BytesIO()
metadata_io.write(yaml.dump(metadata, default_flow_style=False).encode())
print(f'writing metadata to file {metadata_filepath}')
client.write_data(metadata_filename, metadata_io)
print(f'adding {metadata_filepath} to tantalus')
if not dry_run:
file_resource, file_instance = tantalus_api.add_file(storage_name,
metadata_filepath,
update=True)
dataset = tantalus_api.get('sequencedataset', id=dataset_id)
new_file_resources = set(dataset['file_resources'])
new_file_resources.add(file_resource['id'])
tantalus_api.update('sequencedataset',
id=dataset_id,
file_resources=list(new_file_resources))
def main(**kwargs):
"""
Queries the GSC for WGS bams. Transfers bams to specified storage if
necessary and uploads metadata to tantalus
Args:
ids: (string) a list of internal IDs to query the GSC for
storage: (string) destination storage to transfer bams to
id_type: (string) type of ID specified (either sample or library)
skip_older_than: (string) skip bams older than this date
tag_name: (string) tag name to associate the resulting sequence datasets
with when importing into tantalus
update: (flag) specifies whether metadata in tantalus is
to be updated or not
skip_file_import: (flag) import only new lanes into tantalus
query_only: (flag) only query for the bam paths on the GSC
"""
# Check if this script is being run on thost
# If not, connect to an ssh client to access /projects/files
if socket.gethostname() != "txshah":
ssh_client = connect_to_client("10.9.208.161")
sftp = ssh_client.open_sftp()
else:
sftp = None
# Connect to the Tantalus API
tantalus_api = TantalusApi()
storage = tantalus_api.get_storage(kwargs["storage"])
# Convert the date to the format we want
if kwargs["skip_older_than"]:
skip_older_than = valid_date(kwargs["skip_older_than"])
# Check that an ID type was specified
if not kwargs["id_type"]:
raise Exception("Please specify an ID type (sample or library")
details = []
for identifier in kwargs["ids"]:
# Query the GSC to see if the ID exists
infos = query_gsc(identifier, kwargs["id_type"])
if not infos:
logging.info("No results for {} {}. Skipping import".format(
kwargs["id_type"], identifier))
else:
logging.info("{} {} exists on the GSC".format(
kwargs["id_type"], identifier))
# Get the data from GSC
details = get_gsc_details(
infos,
skip_older_than=kwargs["skip_older_than"],
)
# Import and transfer each file
for detail in details:
# Rename the bams according to internal templates
bam_paths = rename_bam_paths(detail, storage, sftp)
# If the bam path does not exist at the source, skip
# the transfer and import
if not bam_paths["source_bam_path"]:
break
# Skip import if we only wanted to query for paths
if kwargs["query_only"]:
continue
if not kwargs["skip_file_import"]:
# Transfer the bam to the specified storage
transfer_gsc_bams(detail, bam_paths, storage, sftp)
# Add the files to Tantalus
logging.info("Importing {} to Tantalus".format(
bam_paths["tantalus_bam_path"]))
dataset = import_bam(
storage_name=storage["name"],
bam_file_path=bam_paths["tantalus_bam_path"],
sample=detail["sample"],
library=detail["library"],
lane_infos=detail["lane_info"],
read_type=detail["read_type"],
tag_name=kwargs["tag_name"],
update=kwargs["update"])
logging.info(
"Successfully added sequence dataset with ID {}".format(
dataset["id"]))
else:
logging.info("Importing library {} to tantalus".format(
detail["library"]["library_id"]))
library_pk = tantalus_api.get_or_create(
"dna_library",
library_id=detail["library"]["library_id"],
library_type=detail["library"]["library_type"],
index_format=detail["library"]["index_format"])["id"]
#Only add lanes, libraries, and samples to tantalus
logging.info(
"Importing lanes for library {} to tantalus".format(
detail["library"]["library_id"]))
for lane in detail["lane_info"]:
lane = tantalus_api.get_or_create(
"sequencing_lane",
flowcell_id=lane["flowcell_id"],
dna_library=library_pk,
read_type=lane["read_type"],
lane_number=str(lane["lane_number"]),
sequencing_centre="GSC",
sequencing_instrument=lane["sequencing_instrument"])
logging.info(
"Successfully created lane {} in tantalus".format(
lane["id"]))