def load_single(self, sample): variants_mongo.index_variants() vcf_files = sampleinfo_mongo.get_vcf_files() vcf_file = vcf_files[sample] self.__load_sample_variants(sample, vcf_file) self.__log_single_successfully_loaded(sample)
def __parallel_process_vcf_files(self, vcf_files, num_processors): client, db = mongo.get_connection() variants_mongo.drop_variants_index(db) hotspot_mongo.index_hotspot(db) jobs = set() while len(vcf_files) > 0: args = vcf_files.pop(0) sample = args[0] vcf_file = args[1] p = Process(target=self.__load_sample_variants, args=(sample, vcf_file)) jobs.add(p) p.start() if len(jobs) == num_processors: for j in jobs: j.join() jobs.clear() variants_mongo.index_variants(db) client.close()