def collect_aa_variants_from_db(self, sample_acc_ids: list):
all_db_annotations = dict() # acc_id -> [(aa_var1), (aa_var_2), ...]
def do(session):
nonlocal all_db_annotations
all_aa_variants = session \
.query(Sequence.accession_id, Annotation.start, Annotation.stop, Annotation.gene_name,
Annotation.feature_type, Annotation.product,
AminoAcidVariant.start_aa_original, AminoAcidVariant.sequence_aa_original,
AminoAcidVariant.sequence_aa_alternative) \
.select_from(Annotation, AminoAcidVariant, Sequence) \
.filter(Annotation.annotation_id == AminoAcidVariant.annotation_id,
Annotation.sequence_id == Sequence.sequence_id,
Sequence.accession_id.in_(sample_acc_ids)) \
.order_by(Sequence.accession_id, Annotation.start, AminoAcidVariant.start_aa_original) \
.all()
for acc_id, ann_start, ann_stop, gene_name, feature_type, product, \
start_aa_original, sequence_aa_original,sequence_aa_alternative in all_aa_variants:
# group aa_variants by accession id into a map
try:
seq_annotations = all_db_annotations[acc_id]
except KeyError:
seq_annotations = list()
all_db_annotations[acc_id] = seq_annotations
content = (ann_start, ann_stop, gene_name, feature_type, product,
start_aa_original, sequence_aa_original, sequence_aa_alternative)
seq_annotations.append(content)
try_py_function(do)
return all_db_annotations
def generate_fasta(virus_taxon_id: int,
virus_folder_name: str,
generated_file_name: str,
only_null_lineages: bool = False) -> str:
"""
Generates a multi fasta file containing all the sequences of the given virus.
:return: the absolute path to the generated fasta file.
"""
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
virus_db_id = virus_database_id(virus_taxon_id)
if virus_db_id is None:
raise Exception('Before running this algorithm, create the '
f'virus associated with taxon {virus_taxon_id}')
def get_acc_ids_and_sequences_from_db(
session: database.Session) -> Generator[Tuple, None, None]:
query = session.query(database.Sequence.accession_id,
database.NucleotideSequence.nucleotide_sequence)\
.filter(database.Sequence.virus_id == virus_db_id,
database.Sequence.sequence_id == database.NucleotideSequence.sequence_id)
if only_null_lineages:
query = query.filter(database.Sequence.lineage == None)
for pair in query.all():
yield pair[0], pair[1]
def get_total_acc_ids_from_db(session: database.Session) -> int:
query = session.query(func.count(database.Sequence.accession_id)) \
.filter(database.Sequence.virus_id == virus_db_id)
if only_null_lineages:
query = query.filter(database.Sequence.lineage == None)
return query.first()[0]
target_file_path = get_local_folder_for(
virus_folder_name, FileType.Fasta) + generated_file_name
logger.info(f"Generating fasta...")
with open(file=target_file_path, mode='w') as file:
total_count = database.try_py_function(get_total_acc_ids_from_db)
print(total_count)
data = database.try_py_function(get_acc_ids_and_sequences_from_db)
progress = tqdm(total=total_count)
if total_count > 0:
first = next(data)
file.write(f'>{first[0]}\n')
file.write(first[1])
progress.update()
for acc_id_and_sequences in data:
file.write(f'\n>{acc_id_and_sequences[0]}\n')
file.write(acc_id_and_sequences[1])
progress.update()
target_file_path = abspath(target_file_path)
logger.info(f"Fasta file generated at {target_file_path}")
return target_file_path
def _measure_delta_in_db(self,
virus_db_id: int,
is_primary_acc_id: bool,
sources: Optional[List[str]] = None):
self._is_primary_acc_id = is_primary_acc_id
self._virus_db_id = virus_db_id
self._sources = sources
if is_primary_acc_id:
_sequences_in_db_at_start = database.try_py_function(
vcm.sequence_primary_accession_ids, virus_db_id, sources)
else:
_sequences_in_db_at_start = database.try_py_function(
vcm.sequence_alternative_accession_ids, virus_db_id, sources)
self._sequences_in_db_at_start = Counter(_sequences_in_db_at_start)
def virus_database_id(virus_taxon_id) -> Optional[int]:
class Virus:
@staticmethod
def taxon_id():
return virus_taxon_id
def get_virus_db_id(session) -> Optional[int]:
db_virus = vcm.get_virus(session, Virus())
return db_virus.virus_id if db_virus else None
return database.try_py_function(get_virus_db_id)
def get_sequences_in_current_data(self) -> dict:
def do(session):
all_sequences = dict()
for db_items in session.query(Sequence, HostSample, SequencingProject) \
.filter(Sequence.strain_name.isnot(None),
Sequence.host_sample_id == HostSample.host_sample_id,
Sequence.sequencing_project_id == SequencingProject.sequencing_project_id) \
.yield_per(100):
source_seq, source_host, source_prj = db_items
all_sequences[source_seq.accession_id] = (source_seq.strain_name, int(source_seq.length),
source_seq.gc_percentage, source_seq.n_percentage,
source_host.collection_date, source_host.originating_lab,
str(source_prj.submission_date) if source_prj.submission_date is not None else None,
source_prj.sequencing_lab,
source_host.country, source_host.region,
source_host.isolation_source)
return all_sequences
return try_py_function(do)
def get_sequences_in_current_data() -> dict:
def do(session):
all_sequences = dict()
for db_items in session.query(Sequence, HostSample, NucleotideSequence) \
.select_from(Sequence, HostSample, SequencingProject, NucleotideSequence) \
.filter(Sequence.strain_name.isnot(None),
Sequence.host_sample_id == HostSample.host_sample_id,
Sequence.sequence_id == NucleotideSequence.sequence_id,
Sequence.sequencing_project_id == SequencingProject.sequencing_project_id,
SequencingProject.database_source == 'COG-UK') \
.yield_per(100):
source_seq, source_host, nuc_seq_db_obj = db_items
all_sequences[source_seq.accession_id] = (
source_seq.length, source_seq.gc_percentage,
source_seq.n_percentage, source_host.collection_date,
source_host.country, source_host.region,
nuc_seq_db_obj.nucleotide_sequence)
return all_sequences
return try_py_function(do)
def import_epitopes(virus_taxon_id: int):
# remember to update the data source
logger.warning(
"Keep in mind to update the data source with 'python main.py download epitopes' before performing"
" the current action. This program will resume in 10 seconds.")
try:
sleep(10)
except KeyboardInterrupt:
return
# begin
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
if virus_taxon_id in [2010960, 186539]: # == bombali or reston ebolavirus
logger.info(f'No epitopes available for virus {virus_taxon_id}.')
return
virus_db_id = virus_database_id(virus_taxon_id)
if virus_db_id is None:
raise Exception(
'Epitopes must be associated to a Virus DB entity. Before running epitopes, create the '
f'virus associated with taxon {virus_taxon_id}')
if epitopes_already_imported(virus_db_id):
logger.info(
'Epitopes for this virus are already imported into the DB. Aborting import.'
)
return
# run epitopes' code
logger.debug(
f'calling epitopes for virus taxon {virus_taxon_id} associated to DB virud_id {virus_db_id}'
)
epitopes, fragments = epitopes_for_virus_taxon(virus_taxon_id)
# write to file
# epitopes_file = open(f'.{sep}epitopes.csv', mode='w')
# epitopes_file.write('virus_db_id\thost_specie_db_id\thost_name\thost_iri\tprotein_ncbi_id\tcell_type\tmhc_class\t'
# 'mhc_allele\tresponse_frequency_positive\tassay_type\tseq\tstart\tstop\text_links\t'
# 'prediction_process\tis_linear\tepitope_iri\tiedb_epitope_id\n')
# epitopes_fragm_file = open(f'.{sep}epitopes_fragments.csv', mode='w')
# epitopes_fragm_file.write('damianos_epitope_id\tseq\tstart\tstop\n')
def do(session: database.Session):
global epitope_id_mappings
try:
for epitope in epitopes:
# get contained values
damianos_epitope_id, virus_taxon_id, host_iri, host_name, host_taxon_id, protein_ncbi_id, cell_type, \
mhc_class, mhc_allele, response_frequency_positive, assay_type, seq, start, stop, ext_links, \
prediction_process, is_linear, epitope_iri, iedb_epitope_id = epitope
# put host specie foreign key
host_specie_db_id = create_or_get_host_specie_db_id(
session, host_taxon_id)
# insert epitope in the DB
epitope = (virus_db_id, host_specie_db_id, host_name, host_iri,
protein_ncbi_id, cell_type, mhc_class, mhc_allele,
response_frequency_positive, assay_type, seq, start,
stop, ext_links, prediction_process, is_linear,
epitope_iri, iedb_epitope_id)
# write to file
# types = (str(type(i)) for i in epitope)
# items = (str(i) for i in epitope)
# for i in zip(items, types):
# epitopes_file.write(f'{i[0], i[1]}\t')
# epitopes_file.write('\n')
epitope_db_id = vcm.create_epitope(session, epitope)
# bind epitope ids from Damianos with the ones returned from database
epitope_id_mappings[damianos_epitope_id] = epitope_db_id
for fragment in fragments:
_, damianos_epitope_id, seq, start, stop = fragment
# bind epitope ids from Damianos with the ones returned from database
try:
epitope_db_id = epitope_id_mappings[damianos_epitope_id]
except KeyError as e:
raise KeyError(
f'the epitope fragment ID {damianos_epitope_id} does not appear in the epitope IDs. This epitope fragment'
f' will be not inserted into the DB.')
fragment = (epitope_db_id, seq, start, stop)
# write to file
# types = (str(type(i)) for i in fragment)
# items = (str(i) for i in fragment)
# for i in zip(items, types):
# epitopes_fragm_file.write(f'{i[0], i[1]}\t')
# epitopes_fragm_file.write('\n')
vcm.create_epitope_fragment(session, fragment)
vcm.DBCache.commit_changes()
except Exception as e:
logger.exception(
'Exception occurred while computing and importing epitopes. Epitopes won\'t be inserted into the DB.'
)
vcm.DBCache.rollback_changes()
raise database.RollbackAndRaise(e)
# finally:
# epitopes_file.close()
# epitopes_fragm_file.close()
database.try_py_function(do)
# insert one row for each linear epitope into epitope_fragment table
database.run_script(
f".{sep}sql_scripts{sep}insert_linear_epitopes_into_epi_fragments.sql")
def epitopes_already_imported(virus_db_id):
return database.try_py_function(vcm.check_existence_epitopes, virus_db_id)
def import_samples_into_vcm():
global fasta_list, refseq_sc1, refseq_sc2, refseq_sc1_len, refseq_sc2_len, cached_taxonomy, fasta_folder, \
taxonomy_folder, imported_viruses
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
fasta_folder = get_local_folder_for('NMDC', FileType.SequenceOrSampleData)
taxonomy_folder = get_local_folder_for('NMDC', FileType.TaxonomyData)
fasta_list = get_fasta_list()
stats_module.schedule_samples(set_info=stats_module.StatsBasedOnIds(
[x.rstrip('.fasta') for x in fasta_list], True))
logger.warning(
f'{len(fasta_list)} files found at {base_url}. Some of them may be skipped because they have not metadata'
f' or because they are not realted to SARS or SARS-coV2.')
logger.info(f'downloading fasta files')
download_fastas()
# REFRENCE SEQUENCE
refseq_sc2 = reference_sequence(
"(txid2697049[Organism]) AND srcdb_refseq[Properties]")
refseq_sc2_len = len(refseq_sc2)
refseq_sc1 = reference_sequence(
"txid694009[Organism:exp] NOT txid2697049[Organism] AND srcdb_refseq[Properties]"
)
refseq_sc1_len = len(refseq_sc1)
def virus_taxonomy_pipeline(session: database, taxon: AnyNCBITaxon):
return vcm.create_or_get_virus(session, taxon)
# noinspection PyTypeChecker
def metadata_pipeline(session: database.Session,
a_sample: NMDCVirusSample):
try:
experiment_id = vcm.create_or_get_experiment(session, a_sample)
host_specie_id = vcm.create_or_get_host_specie(session, a_sample)
host_sample_id = vcm.create_or_get_host_sample(
session, a_sample, host_specie_id)
sequencing_project_id = vcm.create_or_get_sequencing_project(
session, a_sample)
sequence, nucleotide_seq = vcm.create_and_get_sequence(
session, a_sample, virus_id, experiment_id, host_sample_id,
sequencing_project_id)
vcm.DBCache.commit_changes()
return sequence.sequence_id
except Exception as e:
if str(e).startswith(
'duplicate key value violates unique constraint "sequence_accession_id_key"'
):
logger.error(
f'exception occurred while working on virus sample {a_sample}: {str(e)}'
)
else:
logger.exception(
f'exception occurred while working on virus sample {a_sample}'
)
vcm.DBCache.rollback_changes()
raise database.Rollback()
def nucleotide__annotations__pipeline(session: database.Session,
a_sample: NMDCVirusSample,
db_sequence_id):
if a_sample.taxon_name(
) == 'Severe acute respiratory syndrome coronavirus 2':
refseq = refseq_sc2
elif a_sample.taxon_name() == 'Bat SARS-related coronavirus':
refseq = refseq_sc1
else:
raise Exception(f'unknown taxon organism {a_sample.taxon_name()}')
try:
file_path = get_local_folder_for(
'NMDC', FileType.Annotations) + str(
sample.primary_accession_number()) + ".pickle"
if not os.path.exists(file_path):
annotations_and_nuc_variants = sequence_aligner(
db_sequence_id, refseq, a_sample.nucleotide_sequence(),
sc2_chr_name, sc2_ann_file_path, sc2_snpeff_db_name)
with open(file_path, mode='wb') as cache_file:
pickle.dump(annotations_and_nuc_variants,
cache_file,
protocol=pickle.HIGHEST_PROTOCOL)
else:
with open(file_path, mode='rb') as cache_file:
annotations_and_nuc_variants = pickle.load(cache_file)
annotations, nuc_variants = annotations_and_nuc_variants
for ann in annotations:
vcm.create_annotation_and_amino_acid_variants(
session, db_sequence_id, *ann)
for nuc in nuc_variants:
vcm.create_nuc_variants_and_impacts(session, db_sequence_id,
nuc)
stats_module.completed_sample(sample.primary_accession_number())
except Exception:
logger.exception(
f'exception occurred while working on annotations and nuc_variants of virus sample '
f'{a_sample.primary_accession_number()}. Rollback transaction.'
)
raise database.Rollback()
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'NMDC sars_cov_2 sequences update',
removed_items=0,
changed_items=0)
logger.info('begin import of selected records')
vcm.DBCache.commit_changes()
total_sequences_imported = 0
total_sequences_skipped = 0
log_of_gisaid_id_path = f"{get_local_folder_for('NMDC', FileType.Logs)}{os.path.sep}gisa_ids.txt"
with open(log_of_gisaid_id_path, mode='w') as log_of_gisaid_id:
for file in tqdm(fasta_list):
try:
sample = NMDCVirusSample(file)
# filter samples by organism
organism_name = sample.taxon_name()
if organism_name != 'Severe acute respiratory syndrome coronavirus 2':
logger.info(
f'Sample {file} skipped because related to organims {organism_name}'
)
total_sequences_skipped += 1
continue
# download taxonomy for new organisms
organism = cached_taxonomy.get('organism_name')
if not organism:
organism_file = download_ncbi_taxonomy_as_xml_from_name(
taxonomy_folder, organism_name)
organism = AnyNCBITaxon(organism_file)
cached_taxonomy[organism_name] = organism
except FileNotFoundError:
logger.error(f'Sample {file} skipped')
total_sequences_skipped += 1
continue
except AssertionError:
logger.exception(f'Sample {file} skipped')
total_sequences_skipped += 1
continue
# virus id associated to this sample
virus_id = database.try_py_function(virus_taxonomy_pipeline,
organism)
if virus_id not in imported_viruses:
imported_viruses.add(virus_id)
database.try_py_function(vcm.update_db_metadata, virus_id,
'NMDC')
vcm.DBCache.commit_changes()
if virus_id:
gisa_id = sample.gisa_id()
if gisa_id:
log_of_gisaid_id.write(gisa_id + '\n')
# import sample
sequence_id = database.try_py_function(metadata_pipeline,
sample)
vcm.DBCache.commit_changes()
if sequence_id:
database.try_py_function(nucleotide__annotations__pipeline,
sample, sequence_id)
total_sequences_imported += 1
logger.info(f'{total_sequences_imported} sequences imported.')
logger.info(f'{total_sequences_skipped} sequences skipped.')
if total_sequences_skipped > 100:
send_message(
f"NMDC importer can have a bug. {total_sequences_skipped} out of "
f"{total_sequences_skipped+total_sequences_imported} were not imported."
)
logger.info(f'list of sequences with GISAID references at path: ' +
log_of_gisaid_id_path)
pipeline_event.added_items = total_sequences_imported
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)
def check_samples_imported(self):
# overwrite Value object with int for a simpler handling
self._number_completed_samples = self._number_completed_samples.value
self._number_removed_samples = self._number_removed_samples.value
message = f"\n" \
f"STATS MODULE:\n" \
f"Scheduled import of {self._number_scheduled_samples} samples\n" \
f"Removed {self._number_removed_samples} samples\n" \
f"Completed {self._number_completed_samples} samples\n"
warn = False
# find errors
scheduled_not_completed = self._number_scheduled_samples - self._number_completed_samples
completed_not_scheduled = self._number_completed_samples - self._number_scheduled_samples
if scheduled_not_completed > 0:
warn = True
message += f'Failed (scheduled but not completed) {scheduled_not_completed} samples\n'
if completed_not_scheduled > 0:
warn = True
message += f'Wrongly processed (completed but not scheduled) {completed_not_scheduled} samples\n'
# check against DB
if self._virus_db_id is not None:
current_sequences_in_db = database.try_py_function(
vcm.sequence_primary_accession_ids, self._virus_db_id,
self._sources)
current_sequences_in_db = Counter(current_sequences_in_db)
inserted_in_db = current_sequences_in_db - self._sequences_in_db_at_start
removed_from_db = self._sequences_in_db_at_start - current_sequences_in_db
message += f"Sequences in DB:\n" \
f"\tprevious number {sum(self._sequences_in_db_at_start.values())}.\n" \
f"\tcurrent number {sum(current_sequences_in_db.values())}\n" \
f"\tdifference: {sum(inserted_in_db.values())} new - {sum(removed_from_db.values())} missing from source and deleted = {sum(inserted_in_db.values()) - sum(removed_from_db.values())}\n"
# check duplicates
num_duplcates_at_start = sum(
self._sequences_in_db_at_start.values()) - len(
set(self._sequences_in_db_at_start))
num_current_duplicates = sum(
current_sequences_in_db.values()) - len(
set(current_sequences_in_db))
if num_duplcates_at_start > 0 or num_current_duplicates > 0:
warn = True
message += f"Duplicated accession_ids in DB:\n" \
f"\tprevious number of duplicates: {num_duplcates_at_start}\n" \
f"\tcurrent number of duplicates: {num_current_duplicates}\n" \
f"\tdetail of current duplicated accession_ids: {sorted(list(current_sequences_in_db - Counter(set(current_sequences_in_db))))}\n"
num_completed_not_inserted = self._number_completed_samples - sum(
inserted_in_db.values()) - self._number_removed_samples
if num_completed_not_inserted > 0:
warn = True
message += f'Number of samples completed and not imported (or imported twice) into the DB: {num_completed_not_inserted}\n'
elif num_completed_not_inserted < 0:
warn = True
message += f'Number of samples imported into the DB but incomplete: {-num_completed_not_inserted}\n'
# performance
message += self._performance_message(
self._number_completed_samples)
if not warn:
logger.info(message)
else:
logger.warning(message)
def check_samples_imported(self):
if self._completed_samples_acc_id is None:
logger.error(
'STATS MODULE: check_samples_imported called before add_samples. Stats cannot be produced.'
)
else:
warn = False
message = f"\n" \
f"STATS MODULE:\n"
_scheduled_samples_acc_id = set(self._scheduled_samples_acc_id)
message += f"Scheduled import of {len(_scheduled_samples_acc_id)} samples\n"
completed_samples_acc_id = self._queue_to_set(
self._completed_samples_acc_id)
removed_samples_acc_id = self._queue_to_set(
self._removed_samples_acc_id)
self._completed_samples_acc_id.close()
self._removed_samples_acc_id.close()
message += f"Removed {len(removed_samples_acc_id)} samples\n"
message += f"Completed {len(completed_samples_acc_id)} samples\n"
# find errors
scheduled_not_completed = _scheduled_samples_acc_id - completed_samples_acc_id
completed_not_scheduled = completed_samples_acc_id - _scheduled_samples_acc_id
if len(scheduled_not_completed) > 0:
warn = True
message += f'Failed (scheduled but not completed) {len(scheduled_not_completed)} samples\n' \
f'\tsamples id: {scheduled_not_completed}\n'
if len(completed_not_scheduled) > 0:
warn = True
message += f'Wrongly processed (completed but not scheduled) {len(completed_not_scheduled)} samples\n' \
f'\tsamples id: {completed_not_scheduled}\n'
# check against DB
if self._virus_db_id is not None:
if self._is_primary_acc_id:
current_sequences_in_db = database.try_py_function(
vcm.sequence_primary_accession_ids, self._virus_db_id,
self._sources)
else:
current_sequences_in_db = database.try_py_function(
vcm.sequence_alternative_accession_ids,
self._virus_db_id, self._sources)
current_sequences_in_db = Counter(current_sequences_in_db)
inserted_in_db = current_sequences_in_db - self._sequences_in_db_at_start
removed_from_db = self._sequences_in_db_at_start - current_sequences_in_db
message += f"Sequences in DB:\n" \
f"\tprevious number {sum(self._sequences_in_db_at_start.values())}.\n" \
f"\tcurrent number {sum(current_sequences_in_db.values())}\n" \
f"\tdifference: {sum(inserted_in_db.values())} new - {sum(removed_from_db.values())} missing from source and deleted = {sum(inserted_in_db.values()) - sum(removed_from_db.values())}\n"
# check duplicates
num_duplicates_at_start = sum(
self._sequences_in_db_at_start.values()) - len(
set(self._sequences_in_db_at_start))
num_current_duplicates = sum(
current_sequences_in_db.values()) - len(
set(current_sequences_in_db))
if num_duplicates_at_start > 0 or num_current_duplicates > 0:
warn = True
message += f"Duplicated accession_ids in DB:\n" \
f"\tprevious number of duplicates: {num_duplicates_at_start}\n" \
f"\tcurrent number of duplicates: {num_current_duplicates}\n" \
f"\tdetail of current duplicated accession_ids: {sorted(list(current_sequences_in_db - Counter(set(current_sequences_in_db))))}\n"
# check errors in insertions
changed_in_db = Counter(
removed_samples_acc_id
) # this set cannot be retrieved from the database
completed_not_inserted = Counter(
completed_samples_acc_id) - inserted_in_db - changed_in_db
inserted_not_completed = inserted_in_db - Counter(
completed_samples_acc_id) - changed_in_db
if sum(completed_not_inserted.values()) > 0:
warn = True
message += f'Number of samples completed and not imported into the DB: {sum(completed_not_inserted.values())}\n' \
f'\taccession ids: {sorted(list(completed_not_inserted.elements()))}\n'
if sum(inserted_not_completed.values()) < 0:
warn = True
message += f'Number of samples imported into the DB but incomplete: {sum(inserted_not_completed.values())}\n' \
f'\taccession ids: {sorted(list(inserted_not_completed.elements()))}\n'
# performance
message += self._performance_message(len(completed_samples_acc_id))
if not warn:
logger.info(message)
else:
logger.warning(message)
def run(from_sample: Optional[int] = None, to_sample: Optional[int] = None):
global virus, virus_id, import_method
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
virus = COGUKSarsCov2()
# IMPORT VIRUS TAXON DATA
virus_id = database.try_py_function(vcm.create_or_get_virus, virus)
# update last import date
database.try_py_function(vcm.update_db_metadata, virus_id, 'COG-UK')
# find outdated and new samples from source (some sequences can be updated, so the sets are not necessarily disjoint)
logger.warning(
"Current implementation of deltas for COG-UK uses more than 10 GB of RAM to cache query results and save time.\n"
"IF YOUR SYSTEM CAN'T PROVIDE MORE THAN 10 GB OF RAM, STOP THE PROCESS NOW.\n"
"The program will resume in 15 seconds")
try:
sleep(15)
except KeyboardInterrupt:
return
id_outdated_sequences, id_new_sequences = virus.deltas()
logger.warning('Check deltas.. The program will resume in 30 seconds.')
try:
sleep(30)
except KeyboardInterrupt:
return
# select range
if from_sample is not None and to_sample is not None:
id_new_sequences = {
id_new_sequences.pop()
for i in range(to_sample - from_sample)
}
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'COGUK sars_cov_2 sequences update',
removed_items=len(id_outdated_sequences),
changed_items=0,
added_items=len(
id_new_sequences
), # may eventually change if some sequence are not imported
)
# initialize statistics module
stats_module.schedule_samples(
stats_module.StatsBasedOnIds(id_new_sequences, True, virus_id,
['COG-UK']))
# remove outdated sequences
logger.info(f'removing outdated sequences')
database.try_py_function(
vcm.remove_sequence_and_meta_list,
primary_sequence_accession_id=id_outdated_sequences)
stats_module.removed_samples(id_outdated_sequences)
for _id in id_outdated_sequences:
file_path = get_local_folder_for(
virus.name, FileType.Annotations) + str(_id).replace(
'/', '-') + ".pickle"
remove_file(file_path)
# prepare multiprocessing
logger.info(f'importing virus sequences and related tables')
import_method = Parallel()
vcm.DBCache.commit_changes()
for s in virus.get_sequences_of_updated_source(
filter_accession_ids=id_new_sequences):
if not s.nucleotide_sequence():
logger.info(
f'sample {s.primary_accession_number()} skipped because nucleotide sequence is empty or null'
)
continue
try:
database.try_py_function(import_method.import_virus_sample, s)
vcm.DBCache.commit_changes()
except:
logger.exception(
f'exception occurred while working on virus sample {s.primary_accession_number()}'
)
vcm.DBCache.rollback_changes()
logger.info('main process completed')
import_method.tear_down()
# remove leftovers of failed samples
try:
metadata_samples_to_remove: set = stats_module.get_scheduled_not_completed(
)
if len(metadata_samples_to_remove) > 1100:
send_message(
f"COGUK importer can have a bug. {len(metadata_samples_to_remove)} out of "
f"{len(id_new_sequences)} failed.")
pipeline_event.added_items = pipeline_event.added_items - len(
metadata_samples_to_remove)
if len(metadata_samples_to_remove) > 0:
logger.info(
f"Removing metadata leftovers of imports that failed during variant/annotation calling or metadata"
f" ({len(metadata_samples_to_remove)} samples)")
metadata_samples_to_remove_as_string: list = [
str(x) for x in metadata_samples_to_remove
]
logger.trace("Accession id of failed imports:\n"
f"{metadata_samples_to_remove_as_string}")
logger.info("Deleting leftovers in database")
database.try_py_function(vcm.remove_sequence_and_meta_list,
primary_sequence_accession_id=
metadata_samples_to_remove_as_string)
except:
logger.exception(
"Removal of metadata leftovers in the DB of the samples that failed was not successful."
)
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)
def run(from_sample: Optional[int] = None, to_sample: Optional[int] = None):
db_params: dict = import_config.get_database_config_params()
database.config_db_engine(db_params["db_name"], db_params["db_user"],
db_params["db_psw"], db_params["db_port"])
# IMPORT VIRUS TAXON DATA
virus = GISAIDSarsCov2()
virus_id = database.try_py_function(vcm.create_or_get_virus, virus)
database.try_py_function(vcm.update_db_metadata, virus_id, 'GISAID')
# COMPUTE DELTAS
acc_ids_sequences_to_remove, acc_id_sequences_to_import, sequences_to_update = virus.deltas(
)
acc_ids_sequences_to_update = sequences_to_update.keys()
logger.warning('Check deltas.. The program will resume in 30 seconds.')
sleep(30)
# MIND FROM_SAMPLE/TO_SAMPLE
if from_sample is not None and to_sample is not None:
count_el_to_import = abs(to_sample - from_sample)
count_el_to_import = min(count_el_to_import,
len(acc_id_sequences_to_import))
count_el_to_ignore = len(
acc_id_sequences_to_import) - count_el_to_import
try:
for i in range(count_el_to_ignore):
acc_id_sequences_to_import.pop()
except KeyError:
pass # if pop on empty set
# create pipeline_event (will be inserted later)
pipeline_event = database.PipelineEvent(
event_date=datetime.now().strftime("%Y-%m-%d"),
event_name=f'GISAID sars_cov_2 sequences update',
removed_items=len(acc_ids_sequences_to_remove),
changed_items=len(
acc_ids_sequences_to_update
), # provisional - (sqlalchemy wants a value at obj creation)
added_items=len(
acc_id_sequences_to_import
) # provisional - (sqlalchemy wants a value at obj creation)
)
changed_items = 0
added_items = 0
stats_module.schedule_samples(
stats_module.StatsBasedOnIds(acc_id_sequences_to_import, True,
virus_id, ['GISAID']))
logger.info('Removing outdated sequences...')
# REMOVE OUTDATED SEQUENCES
database.try_py_function(vcm.remove_sequence_and_meta_list,
acc_ids_sequences_to_remove, None)
stats_module.removed_samples(acc_ids_sequences_to_remove)
# IMPORT NEW/CHANGED SEQUENCES
vcm.DBCache.commit_changes()
logger.info(f'Importing virus sequences and related tables...')
import_method = Sequential(virus_id)
progress = tqdm(total=len(acc_id_sequences_to_import) +
len(acc_ids_sequences_to_update))
for sample in virus.get_sequences_of_updated_source():
sample_accession_id = sample.primary_accession_number()
try:
if sample_accession_id in acc_id_sequences_to_import:
# import sample from scratch
database.try_py_function(import_method.import_virus_sample,
sample)
vcm.DBCache.commit_changes()
progress.update()
added_items += 1
elif sample_accession_id in acc_ids_sequences_to_update:
# update values inside the database
changes_in_sequence = sequences_to_update[sample_accession_id]
database.try_py_function(import_method.update_virus_sample,
sample, changes_in_sequence)
vcm.DBCache.commit_changes()
progress.update()
changed_items += 1
except KeyboardInterrupt:
logger.info("main loop interrupted by the user")
break
except:
logger.exception(
f'exception occurred while working on virus sample {sample.internal_id()}'
)
vcm.DBCache.rollback_changes()
logger.info('main loop completed')
import_method.tear_down()
logger.info('Removal of unused database objects...')
database.try_py_function(vcm.clean_objects_unreachable_from_sequences)
if len(acc_id_sequences_to_import) - added_items > 100:
send_message(
f"GISAID importer can have a bug. import of {len(acc_id_sequences_to_import) - added_items} out of"
f" {len(acc_id_sequences_to_import)} failed.")
pipeline_event.changed_items = changed_items
pipeline_event.added_items = added_items
database.try_py_function(vcm.insert_data_update_pipeline_event,
pipeline_event)