def generate_fasta(virus_taxon_id: int,
virus_folder_name: str,
generated_file_name: str,
only_null_lineages: bool = False) -> str:
"""
Generates a multi fasta file containing all the sequences of the given virus.
:return: the absolute path to the generated fasta file.
"""
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
virus_db_id = virus_database_id(virus_taxon_id)
if virus_db_id is None:
raise Exception('Before running this algorithm, create the '
f'virus associated with taxon {virus_taxon_id}')
def get_acc_ids_and_sequences_from_db(
session: database.Session) -> Generator[Tuple, None, None]:
query = session.query(database.Sequence.accession_id,
database.NucleotideSequence.nucleotide_sequence)\
.filter(database.Sequence.virus_id == virus_db_id,
database.Sequence.sequence_id == database.NucleotideSequence.sequence_id)
if only_null_lineages:
query = query.filter(database.Sequence.lineage == None)
for pair in query.all():
yield pair[0], pair[1]
def get_total_acc_ids_from_db(session: database.Session) -> int:
query = session.query(func.count(database.Sequence.accession_id)) \
.filter(database.Sequence.virus_id == virus_db_id)
if only_null_lineages:
query = query.filter(database.Sequence.lineage == None)
return query.first()[0]
target_file_path = get_local_folder_for(
virus_folder_name, FileType.Fasta) + generated_file_name
logger.info(f"Generating fasta...")
with open(file=target_file_path, mode='w') as file:
total_count = database.try_py_function(get_total_acc_ids_from_db)
print(total_count)
data = database.try_py_function(get_acc_ids_and_sequences_from_db)
progress = tqdm(total=total_count)
if total_count > 0:
first = next(data)
file.write(f'>{first[0]}\n')
file.write(first[1])
progress.update()
for acc_id_and_sequences in data:
file.write(f'\n>{acc_id_and_sequences[0]}\n')
file.write(acc_id_and_sequences[1])
progress.update()
target_file_path = abspath(target_file_path)
logger.info(f"Fasta file generated at {target_file_path}")
return target_file_path
def import_epitopes(virus_taxon_id: int):
# remember to update the data source
logger.warning(
"Keep in mind to update the data source with 'python main.py download epitopes' before performing"
" the current action. This program will resume in 10 seconds.")
try:
sleep(10)
except KeyboardInterrupt:
return
# begin
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
if virus_taxon_id in [2010960, 186539]: # == bombali or reston ebolavirus
logger.info(f'No epitopes available for virus {virus_taxon_id}.')
return
virus_db_id = virus_database_id(virus_taxon_id)
if virus_db_id is None:
raise Exception(
'Epitopes must be associated to a Virus DB entity. Before running epitopes, create the '
f'virus associated with taxon {virus_taxon_id}')
if epitopes_already_imported(virus_db_id):
logger.info(
'Epitopes for this virus are already imported into the DB. Aborting import.'
)
return
# run epitopes' code
logger.debug(
f'calling epitopes for virus taxon {virus_taxon_id} associated to DB virud_id {virus_db_id}'
)
epitopes, fragments = epitopes_for_virus_taxon(virus_taxon_id)
# write to file
# epitopes_file = open(f'.{sep}epitopes.csv', mode='w')
# epitopes_file.write('virus_db_id\thost_specie_db_id\thost_name\thost_iri\tprotein_ncbi_id\tcell_type\tmhc_class\t'
# 'mhc_allele\tresponse_frequency_positive\tassay_type\tseq\tstart\tstop\text_links\t'
# 'prediction_process\tis_linear\tepitope_iri\tiedb_epitope_id\n')
# epitopes_fragm_file = open(f'.{sep}epitopes_fragments.csv', mode='w')
# epitopes_fragm_file.write('damianos_epitope_id\tseq\tstart\tstop\n')
def do(session: database.Session):
global epitope_id_mappings
try:
for epitope in epitopes:
# get contained values
damianos_epitope_id, virus_taxon_id, host_iri, host_name, host_taxon_id, protein_ncbi_id, cell_type, \
mhc_class, mhc_allele, response_frequency_positive, assay_type, seq, start, stop, ext_links, \
prediction_process, is_linear, epitope_iri, iedb_epitope_id = epitope
# put host specie foreign key
host_specie_db_id = create_or_get_host_specie_db_id(
session, host_taxon_id)
# insert epitope in the DB
epitope = (virus_db_id, host_specie_db_id, host_name, host_iri,
protein_ncbi_id, cell_type, mhc_class, mhc_allele,
response_frequency_positive, assay_type, seq, start,
stop, ext_links, prediction_process, is_linear,
epitope_iri, iedb_epitope_id)
# write to file
# types = (str(type(i)) for i in epitope)
# items = (str(i) for i in epitope)
# for i in zip(items, types):
# epitopes_file.write(f'{i[0], i[1]}\t')
# epitopes_file.write('\n')
epitope_db_id = vcm.create_epitope(session, epitope)
# bind epitope ids from Damianos with the ones returned from database
epitope_id_mappings[damianos_epitope_id] = epitope_db_id
for fragment in fragments:
_, damianos_epitope_id, seq, start, stop = fragment
# bind epitope ids from Damianos with the ones returned from database
try:
epitope_db_id = epitope_id_mappings[damianos_epitope_id]
except KeyError as e:
raise KeyError(
f'the epitope fragment ID {damianos_epitope_id} does not appear in the epitope IDs. This epitope fragment'
f' will be not inserted into the DB.')
fragment = (epitope_db_id, seq, start, stop)
# write to file
# types = (str(type(i)) for i in fragment)
# items = (str(i) for i in fragment)
# for i in zip(items, types):
# epitopes_fragm_file.write(f'{i[0], i[1]}\t')
# epitopes_fragm_file.write('\n')
vcm.create_epitope_fragment(session, fragment)
vcm.DBCache.commit_changes()
except Exception as e:
logger.exception(
'Exception occurred while computing and importing epitopes. Epitopes won\'t be inserted into the DB.'
)
vcm.DBCache.rollback_changes()
raise database.RollbackAndRaise(e)
# finally:
# epitopes_file.close()
# epitopes_fragm_file.close()
database.try_py_function(do)
# insert one row for each linear epitope into epitope_fragment table
database.run_script(
f".{sep}sql_scripts{sep}insert_linear_epitopes_into_epi_fragments.sql")
def import_samples_into_vcm():
global fasta_list, refseq_sc1, refseq_sc2, refseq_sc1_len, refseq_sc2_len, cached_taxonomy, fasta_folder, \
taxonomy_folder, imported_viruses
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
fasta_folder = get_local_folder_for('NMDC', FileType.SequenceOrSampleData)
taxonomy_folder = get_local_folder_for('NMDC', FileType.TaxonomyData)
fasta_list = get_fasta_list()
stats_module.schedule_samples(set_info=stats_module.StatsBasedOnIds(
[x.rstrip('.fasta') for x in fasta_list], True))
logger.warning(
f'{len(fasta_list)} files found at {base_url}. Some of them may be skipped because they have not metadata'
f' or because they are not realted to SARS or SARS-coV2.')
logger.info(f'downloading fasta files')
download_fastas()
# REFRENCE SEQUENCE
refseq_sc2 = reference_sequence(
"(txid2697049[Organism]) AND srcdb_refseq[Properties]")
refseq_sc2_len = len(refseq_sc2)
refseq_sc1 = reference_sequence(
"txid694009[Organism:exp] NOT txid2697049[Organism] AND srcdb_refseq[Properties]"
)
refseq_sc1_len = len(refseq_sc1)
def virus_taxonomy_pipeline(session: database, taxon: AnyNCBITaxon):
return vcm.create_or_get_virus(session, taxon)
# noinspection PyTypeChecker
def metadata_pipeline(session: database.Session,
a_sample: NMDCVirusSample):
try:
experiment_id = vcm.create_or_get_experiment(session, a_sample)
host_specie_id = vcm.create_or_get_host_specie(session, a_sample)
host_sample_id = vcm.create_or_get_host_sample(
session, a_sample, host_specie_id)
sequencing_project_id = vcm.create_or_get_sequencing_project(
session, a_sample)
sequence, nucleotide_seq = vcm.create_and_get_sequence(
session, a_sample, virus_id, experiment_id, host_sample_id,
sequencing_project_id)
vcm.DBCache.commit_changes()
return sequence.sequence_id
except Exception as e:
if str(e).startswith(
'duplicate key value violates unique constraint "sequence_accession_id_key"'
):
logger.error(
f'exception occurred while working on virus sample {a_sample}: {str(e)}'
)
else:
logger.exception(
f'exception occurred while working on virus sample {a_sample}'
)
vcm.DBCache.rollback_changes()
raise database.Rollback()
def nucleotide__annotations__pipeline(session: database.Session,
a_sample: NMDCVirusSample,
db_sequence_id):
if a_sample.taxon_name(
) == 'Severe acute respiratory syndrome coronavirus 2':
refseq = refseq_sc2
elif a_sample.taxon_name() == 'Bat SARS-related coronavirus':
refseq = refseq_sc1
else:
raise Exception(f'unknown taxon organism {a_sample.taxon_name()}')
try:
file_path = get_local_folder_for(
'NMDC', FileType.Annotations) + str(
sample.primary_accession_number()) + ".pickle"
if not os.path.exists(file_path):
annotations_and_nuc_variants = sequence_aligner(
db_sequence_id, refseq, a_sample.nucleotide_sequence(),
sc2_chr_name, sc2_ann_file_path, sc2_snpeff_db_name)
with open(file_path, mode='wb') as cache_file:
pickle.dump(annotations_and_nuc_variants,
cache_file,
protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(file_path, mode='rb') as cache_file:
annotations_and_nuc_variants = pickle.load(cache_file)
annotations, nuc_variants = annotations_and_nuc_variants
for ann in annotations:
vcm.create_annotation_and_amino_acid_variants(
session, db_sequence_id, *ann)
for nuc in nuc_variants:
vcm.create_nuc_variants_and_impacts(session, db_sequence_id,
nuc)
stats_module.completed_sample(sample.primary_accession_number())
except Exception:
logger.exception(
f'exception occurred while working on annotations and nuc_variants of virus sample '
f'{a_sample.primary_accession_number()}. Rollback transaction.'
)
raise database.Rollback()
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'NMDC sars_cov_2 sequences update',
removed_items=0,
changed_items=0)
logger.info('begin import of selected records')
vcm.DBCache.commit_changes()
total_sequences_imported = 0
total_sequences_skipped = 0
log_of_gisaid_id_path = f"{get_local_folder_for('NMDC', FileType.Logs)}{os.path.sep}gisa_ids.txt"
with open(log_of_gisaid_id_path, mode='w') as log_of_gisaid_id:
for file in tqdm(fasta_list):
try:
sample = NMDCVirusSample(file)
# filter samples by organism
organism_name = sample.taxon_name()
if organism_name != 'Severe acute respiratory syndrome coronavirus 2':
logger.info(
f'Sample {file} skipped because related to organims {organism_name}'
)
total_sequences_skipped += 1
continue
# download taxonomy for new organisms
organism = cached_taxonomy.get('organism_name')
if not organism:
organism_file = download_ncbi_taxonomy_as_xml_from_name(
taxonomy_folder, organism_name)
organism = AnyNCBITaxon(organism_file)
cached_taxonomy[organism_name] = organism
except FileNotFoundError:
logger.error(f'Sample {file} skipped')
total_sequences_skipped += 1
continue
except AssertionError:
logger.exception(f'Sample {file} skipped')
total_sequences_skipped += 1
continue
# virus id associated to this sample
virus_id = database.try_py_function(virus_taxonomy_pipeline,
organism)
if virus_id not in imported_viruses:
imported_viruses.add(virus_id)
database.try_py_function(vcm.update_db_metadata, virus_id,
'NMDC')
vcm.DBCache.commit_changes()
if virus_id:
gisa_id = sample.gisa_id()
if gisa_id:
log_of_gisaid_id.write(gisa_id + '\n')
# import sample
sequence_id = database.try_py_function(metadata_pipeline,
sample)
vcm.DBCache.commit_changes()
if sequence_id:
database.try_py_function(nucleotide__annotations__pipeline,
sample, sequence_id)
total_sequences_imported += 1
logger.info(f'{total_sequences_imported} sequences imported.')
logger.info(f'{total_sequences_skipped} sequences skipped.')
if total_sequences_skipped > 100:
send_message(
f"NMDC importer can have a bug. {total_sequences_skipped} out of "
f"{total_sequences_skipped+total_sequences_imported} were not imported."
)
logger.info(f'list of sequences with GISAID references at path: ' +
log_of_gisaid_id_path)
pipeline_event.added_items = total_sequences_imported
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)
def run(from_sample: Optional[int] = None, to_sample: Optional[int] = None):
global virus, virus_id, import_method
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
virus = COGUKSarsCov2()
# IMPORT VIRUS TAXON DATA
virus_id = database.try_py_function(vcm.create_or_get_virus, virus)
# update last import date
database.try_py_function(vcm.update_db_metadata, virus_id, 'COG-UK')
# find outdated and new samples from source (some sequences can be updated, so the sets are not necessarily disjoint)
logger.warning(
"Current implementation of deltas for COG-UK uses more than 10 GB of RAM to cache query results and save time.\n"
"IF YOUR SYSTEM CAN'T PROVIDE MORE THAN 10 GB OF RAM, STOP THE PROCESS NOW.\n"
"The program will resume in 15 seconds")
try:
sleep(15)
except KeyboardInterrupt:
return
id_outdated_sequences, id_new_sequences = virus.deltas()
logger.warning('Check deltas.. The program will resume in 30 seconds.')
try:
sleep(30)
except KeyboardInterrupt:
return
# select range
if from_sample is not None and to_sample is not None:
id_new_sequences = {
id_new_sequences.pop()
for i in range(to_sample - from_sample)
}
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'COGUK sars_cov_2 sequences update',
removed_items=len(id_outdated_sequences),
changed_items=0,
added_items=len(
id_new_sequences
), # may eventually change if some sequence are not imported
)
# initialize statistics module
stats_module.schedule_samples(
stats_module.StatsBasedOnIds(id_new_sequences, True, virus_id,
['COG-UK']))
# remove outdated sequences
logger.info(f'removing outdated sequences')
database.try_py_function(
vcm.remove_sequence_and_meta_list,
primary_sequence_accession_id=id_outdated_sequences)
stats_module.removed_samples(id_outdated_sequences)
for _id in id_outdated_sequences:
file_path = get_local_folder_for(
virus.name, FileType.Annotations) + str(_id).replace(
'/', '-') + ".pickle"
remove_file(file_path)
# prepare multiprocessing
logger.info(f'importing virus sequences and related tables')
import_method = Parallel()
vcm.DBCache.commit_changes()
for s in virus.get_sequences_of_updated_source(
filter_accession_ids=id_new_sequences):
if not s.nucleotide_sequence():
logger.info(
f'sample {s.primary_accession_number()} skipped because nucleotide sequence is empty or null'
)
continue
try:
database.try_py_function(import_method.import_virus_sample, s)
vcm.DBCache.commit_changes()
except:
logger.exception(
f'exception occurred while working on virus sample {s.primary_accession_number()}'
)
vcm.DBCache.rollback_changes()
logger.info('main process completed')
import_method.tear_down()
# remove leftovers of failed samples
try:
metadata_samples_to_remove: set = stats_module.get_scheduled_not_completed(
)
if len(metadata_samples_to_remove) > 1100:
send_message(
f"COGUK importer can have a bug. {len(metadata_samples_to_remove)} out of "
f"{len(id_new_sequences)} failed.")
pipeline_event.added_items = pipeline_event.added_items - len(
metadata_samples_to_remove)
if len(metadata_samples_to_remove) > 0:
logger.info(
f"Removing metadata leftovers of imports that failed during variant/annotation calling or metadata"
f" ({len(metadata_samples_to_remove)} samples)")
metadata_samples_to_remove_as_string: list = [
str(x) for x in metadata_samples_to_remove
]
logger.trace("Accession id of failed imports:\n"
f"{metadata_samples_to_remove_as_string}")
logger.info("Deleting leftovers in database")
database.try_py_function(vcm.remove_sequence_and_meta_list,
primary_sequence_accession_id=
metadata_samples_to_remove_as_string)
except:
logger.exception(
"Removal of metadata leftovers in the DB of the samples that failed was not successful."
)
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)
def run(from_sample: Optional[int] = None, to_sample: Optional[int] = None):
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
# IMPORT VIRUS TAXON DATA
virus = GISAIDSarsCov2()
virus_id = database.try_py_function(vcm.create_or_get_virus, virus)
database.try_py_function(vcm.update_db_metadata, virus_id, 'GISAID')
# COMPUTE DELTAS
acc_ids_sequences_to_remove, acc_id_sequences_to_import, sequences_to_update = virus.deltas(
)
acc_ids_sequences_to_update = sequences_to_update.keys()
logger.warning('Check deltas.. The program will resume in 30 seconds.')
sleep(30)
# MIND FROM_SAMPLE/TO_SAMPLE
if from_sample is not None and to_sample is not None:
count_el_to_import = abs(to_sample - from_sample)
count_el_to_import = min(count_el_to_import,
len(acc_id_sequences_to_import))
count_el_to_ignore = len(
acc_id_sequences_to_import) - count_el_to_import
try:
for i in range(count_el_to_ignore):
acc_id_sequences_to_import.pop()
except KeyError:
pass # if pop on empty set
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'GISAID sars_cov_2 sequences update',
removed_items=len(acc_ids_sequences_to_remove),
changed_items=len(
acc_ids_sequences_to_update
), # provisional - (sqlalchemy wants a value at obj creation)
added_items=len(
acc_id_sequences_to_import
) # provisional - (sqlalchemy wants a value at obj creation)
)
changed_items = 0
added_items = 0
stats_module.schedule_samples(
stats_module.StatsBasedOnIds(acc_id_sequences_to_import, True,
virus_id, ['GISAID']))
logger.info('Removing outdated sequences...')
# REMOVE OUTDATED SEQUENCES
database.try_py_function(vcm.remove_sequence_and_meta_list,
acc_ids_sequences_to_remove, None)
stats_module.removed_samples(acc_ids_sequences_to_remove)
# IMPORT NEW/CHANGED SEQUENCES
vcm.DBCache.commit_changes()
logger.info(f'Importing virus sequences and related tables...')
import_method = Sequential(virus_id)
progress = tqdm(total=len(acc_id_sequences_to_import) +
len(acc_ids_sequences_to_update))
for sample in virus.get_sequences_of_updated_source():
sample_accession_id = sample.primary_accession_number()
try:
if sample_accession_id in acc_id_sequences_to_import:
# import sample from scratch
database.try_py_function(import_method.import_virus_sample,
sample)
vcm.DBCache.commit_changes()
progress.update()
added_items += 1
elif sample_accession_id in acc_ids_sequences_to_update:
# update values inside the database
changes_in_sequence = sequences_to_update[sample_accession_id]
database.try_py_function(import_method.update_virus_sample,
sample, changes_in_sequence)
vcm.DBCache.commit_changes()
progress.update()
changed_items += 1
except KeyboardInterrupt:
logger.info("main loop interrupted by the user")
break
except:
logger.exception(
f'exception occurred while working on virus sample {sample.internal_id()}'
)
vcm.DBCache.rollback_changes()
logger.info('main loop completed')
import_method.tear_down()
logger.info('Removal of unused database objects...')
database.try_py_function(vcm.clean_objects_unreachable_from_sequences)
if len(acc_id_sequences_to_import) - added_items > 100:
send_message(
f"GISAID importer can have a bug. import of {len(acc_id_sequences_to_import) - added_items} out of"
f" {len(acc_id_sequences_to_import)} failed.")
pipeline_event.changed_items = changed_items
pipeline_event.added_items = added_items
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)