def test_fastaReader(): print ("Loading data") now = datetime.datetime.now() path = "../input/hg38/test.fa" faReader = fastaReader(path,name="hg38",useOnlyChromosomes=["chr1"]) faReader = faReader.read_data() print (faReader) print ("Time:",datetime.datetime.now() - now)
def test_hicReader(): genome = fastaReader("../input/hg38/test.fa",name="hg38") faReader = genome.read_data() now = datetime.datetime.now() hic = hicReader(fname="../input/4DNFI2TK7L2F.hic", genome=genome, resolution = 100000) hic = hic.read_data() print (hic.norms) result = hic.get_contact(Interval("chr1",0,120000000)) # single float value or NaN print (result) result = hic.get_chr_contact("chr1") # returns sparse matrix of the whole chrm as pandas dataframe print (datetime.datetime.now() - now)
def calc_sparsity(): logging.basicConfig( level=logging.DEBUG) # set to INFO for less detailed output ### load data ### # load genome chr = "chr2" faReader = fastaReader("../input/hg38/hg38.fa", useOnlyChromosomes=[chr]) faReader = faReader.read_data() # load chipSeq bwReader1 = bigWigReader("../input/ENCFF473IZV_H1_CTCF.bigWig", genome=faReader, inMemory=True) bwReader1 = bwReader1.readData() arr = bwReader1.data[chr] print(len(arr)) nonzero = arr[np.nonzero(arr)] print(len(nonzero)) finite = nonzero[np.isfinite(nonzero)] print(len(finite))
def calc_insulation_around_CTCF(chr, resolution=5000, window_size=20): logging.basicConfig( level=logging.DEBUG) # set to INFO for less detailed output ### load data ### # load genome faReader = fastaReader("../input/hg38/hg38.fa", useOnlyChromosomes=[chr]) faReader = faReader.read_data() # load chipSeq1 bwReader1 = bigWigReader("../input/ENCFF473IZV_H1_CTCF.bigWig", genome=faReader, inMemory=True) bwReader1 = bwReader1.readData() #load contacts hic = hicReader("../input/4DNFI2TK7L2F.hic", genome=faReader, resolution=resolution) hic = hic.read_data() ### run simple check that contact count correlate with ChipSeq signal ### ### generate some random samples #### # get size of the chr1 total_length = faReader.get_chr_sizes()[chr] all_CTCF = bwReader1.get_interval(Interval(chr, 0, total_length)) all_CTCF = np.nan_to_num(all_CTCF) binsize = 1000 bins = np.arange(0, total_length - 1, binsize) sums = [np.sum(all_CTCF[a:a + binsize]) for a in bins] peaks = bins[sums > np.percentile(sums, 90)] with open("../out/test.bed", "w") as fout: for i in peaks: fout.write(chr + "\t" + str(i) + "\t" + str(i + binsize) + "\n")
def calc_corr(chr, resolution=5000, window_size=20): logging.basicConfig( level=logging.DEBUG) # set to INFO for less detailed output ### load data ### # load genome faReader = fastaReader("../input/hg38/hg38.fa", useOnlyChromosomes=[chr]) faReader = faReader.read_data() # load chipSeq1 bwReader1 = bigWigReader("../input/ENCFF473IZV_H1_CTCF.bigWig", genome=faReader, inMemory=True) bwReader1 = bwReader1.readData() #load contacts hic = hicReader("../input/4DNFI2TK7L2F.hic", genome=faReader, resolution=resolution) hic = hic.read_data() ### run simple check that contact count correlate with ChipSeq signal ### ### generate some random samples #### # get size of the chr1 total_length = faReader.get_chr_sizes()[chr] # distance between intercting regions in this particular test, in units of resolution sample_size = 5000 # select random points on chr1 random_points_starts = np.random.random_integers( 0, total_length - window_size, sample_size) random_points_starts = np.array( (random_points_starts // resolution) * resolution, dtype=np.uint64) random_points_ends = random_points_starts + window_size # for each of selected points get contact between this point and (point + window_size*resolution) contacts = [] chipSignals = [] seqSignals = [] now = datetime.datetime.now() # start timer logging.info("Starting data generation") for start, end in zip(random_points_starts, random_points_ends): interval = Interval(chr, start, end) assert window_size >= 5 * resolution window = Interval(chr, start + resolution, end) contact = hic.get_contact(interval) if contact == None: contact = 0 if np.isfinite(contact): # chipSignal = np.concatenate((bwReader1.get_interval(Interval(chr,int(start-resolution),int(start+resolution))), # bwReader1.get_interval( # Interval(chr, int(end - resolution), int(end + resolution))))) chipSignal = bwReader1.get_interval(window) chipSignal = np.nan_to_num(chipSignal) chipSignal = np.sum(chipSignal) if np.isfinite(chipSignal): chipSignals.append(chipSignal) seqSignal = np.sum(faReader.get_interval(interval)) seqSignals.append(seqSignal) contacts.append(contact) logging.info("Time for data generation: " + str(datetime.datetime.now() - now)) from scipy.stats import spearmanr, pearsonr res = [] res.append(spearmanr(np.array(contacts), np.array(chipSignals))[0]) res.append(pearsonr(np.array(contacts), np.array(chipSignals))[0]) res.append(spearmanr(np.array(contacts), np.array(seqSignals))[0]) res.append(pearsonr(np.array(contacts), np.array(seqSignals))[0]) return ("\t".join(list(map(str, res))))
def simple_test(): logging.basicConfig( level=logging.DEBUG) # set to INFO for less detailed output ### load data ### # load genome input_folder = "/home/minja/PycharmProjects/3Dpredictor/nn/input/" faReader = fastaReader(input_folder + "hg38/hg38.fa", useOnlyChromosomes=["chr1"]) faReader = faReader.read_data() # load chipSeq bwReader1 = bigWigReader(input_folder + "ENCFF473IZV_H1_CTCF.bigWig", genome=faReader, inMemory=True) bwReader1 = bwReader1.readData() # load chipSeq bwReader2 = bigWigReader(input_folder + "ENCFF966IHQ.bigWig", genome=faReader, inMemory=False) bwReader2 = bwReader2.readData() #load contacts resolution = 5000 hic = hicReader(input_folder + "4DNFI2TK7L2F.hic", genome=faReader, binsize=resolution, indexedData=True) hic = hic.read_data() ### run simple check that contact count correlate with ChipSeq signal ### ### generate some random samples #### # get size of the chr1 total_length = faReader.get_chr_sizes()["chr1"] window_size = 20 * resolution # distance between intercting regions in this particular test, in units of resolution sample_size = 100000 # select random points on chr1 random_points_starts = np.random.random_integers( 0, total_length - window_size, sample_size) random_points_starts = np.array( (random_points_starts // resolution) * resolution, dtype=np.uint64) random_points_ends = random_points_starts + window_size # for each of selected points get contact between this point and (point + window_size*resolution) contacts = [] chipSignals = [] seqSignals = [] now = datetime.datetime.now() # start timer logging.info("Starting data generation") for start, end in zip(random_points_starts, random_points_ends): interval = Interval("chr1", start, end) contact = hic.get_contact(interval) if contact == None: continue else: chipSignal = np.nansum(bwReader1.get_interval(interval)) if np.isfinite(chipSignal): chipSignals.append(chipSignal) seqSignal = np.sum(faReader.get_interval(interval)) seqSignals.append(seqSignal) contacts.append(contact) logging.info("Time for data generation1: " + str(datetime.datetime.now() - now)) # now = datetime.datetime.now() # chipSignals = [] # seqSignals = [] # contacts = [] # for start,end in zip(random_points_starts,random_points_ends): # interval = Interval("chr1",start,end) # contact = hic.get_contact(interval) # if contact == None: # continue # else: # chipSignal = np.nansum(bwReader2.get_interval(interval)) # if np.isfinite(chipSignal): # chipSignals.append(chipSignal) # seqSignal = np.sum(faReader.get_interval(interval)) # seqSignals.append(seqSignal) # contacts.append(contact) # # logging.info("Time for data generation2: " + str(datetime.datetime.now() - now)) from scipy.stats import spearmanr import matplotlib.pyplot as plt print(contacts) print(chipSignals) print(spearmanr(np.array(contacts), np.array(chipSignals))) print(np.all(np.isfinite(contacts))) print(np.all(np.isfinite(chipSignals))) plt.scatter(contacts, chipSignals) plt.show()
def test_dump(): faReader = fastaReader("../input/hg38/test.fa",useOnlyChromosomes=["chr2"]) faReader = faReader.read_data() print (faReader.get_chr_sizes())
# params.sample_size = end - start params.sample_size = 2 #how many contacts write to file #params.conttype = conttype params.max_cpus = int(args['max_cpus']) params.keep_only_orient = False params.use_only_contacts_with_CTCF = "all_cont" #"all_cont" or "cont_with_CTCF" rearrangement = False # deletion = Interval("chr" + chr_num, start, end) write_all_chrms_in_file = False #set True if you want write training file consisting several chromosomes fill_empty_contacts = False #set True if you want use all contacts in region, without empty contacts logging.getLogger(__name__).debug("Using input folder " + input_folder) # Read contacts data genome = fastaReader(args['path_to_genome'], useOnlyChromosomes=[chromosome]) #str(chr_num)]) genome = genome.read_data() print(genome.data) now = datetime.datetime.now() params.contacts_reader = hicReader(fname=input_folder + "/" + cell_type + "/" + hic_name, genome=genome, binsize=params.binsize) params.contacts_reader = params.contacts_reader.read_data( fill_empty_contacts=fill_empty_contacts, noDump=False) if params.use_only_contacts_with_CTCF == "cont_with_CTCF": params.proportion = 1 params.contacts_reader.use_contacts_with_CTCF(CTCFfile=input_folder+"/" + cell_type+"/CTCF/"+CTCF_file_name, maxdist=params.maxdist, proportion=params.proportion,
params.mindist = params.binsize * 2 + 1 #minimum distance between contacting regions params.maxdist = 1500000 params.sample_size = 500000 #how many contacts write to file params.conttype = conttype params.max_cpus = 11 params.keep_only_orient = False params.use_only_contacts_with_CTCF = "all_cont" #"all_cont"#"cont_with_CTCF"#"#"all_cont"#"cont_with_CTCF " write_all_chrms_in_file = False #set True if you want write training file consisting several chromosomes fill_empty_contacts = True #set True if you want use all contacts in region, without empty contacts logging.getLogger(__name__).debug("Using input folder " + input_folder) # Read contacts data genome = fastaReader(input_folder + "sequence/hg38/hg38.fa", name="hg38", useOnlyChromosomes=["chr3"]) genome = genome.read_data() # print(genome) # print(genome.data.keys()) now = datetime.datetime.now() params.contacts_reader = hicReader(fname=input_folder + "H1/4DNFI2TK7L2F.hic", genome=genome, binsize=1000) # params.contacts_reader = hicReader(fname=input_folder + "H1/control.chr4.50KBhic", genome=genome, binsize=1000) params.contacts_reader = params.contacts_reader.read_data() if params.use_only_contacts_with_CTCF == "cont_with_CTCF": params.proportion = 1 params.contacts_reader.use_contacts_with_CTCF(
params.mindist = params.binsize * 2 + 1 #minimum distance between contacting regions params.maxdist = 1500000 params.sample_size = 500000 #how many contacts write to file params.conttype = conttype params.max_cpus = 11 params.keep_only_orient = False params.use_only_contacts_with_CTCF = "all_cont" #"all_cont"#"cont_with_CTCF"#"#"all_cont"#"cont_with_CTCF " write_all_chrms_in_file = False #set True if you want write training file consisting several chromosomes fill_empty_contacts = False #set True if you want use all contacts in region, without empty contacts logging.getLogger(__name__).debug("Using input folder " + input_folder) # Read contacts data genome = fastaReader(input_folder + "sequence/hg38/hg38.fa", name="hg38", excludeChromosomes=["chrM", "chrY"]) genome = genome.read_data() # print(genome) # print(genome.data.keys()) now = datetime.datetime.now() params.contacts_reader = hicReader(fname=input_folder + "H1/4DNFI2TK7L2F.hic", genome=genome, binsize=1000) # params.contacts_reader = hicReader(fname=input_folder + "H1/control.chr4.50KBhic", genome=genome, binsize=1000) params.contacts_reader = params.contacts_reader.read_data() if params.use_only_contacts_with_CTCF == "cont_with_CTCF": params.proportion = 1 params.contacts_reader.use_contacts_with_CTCF(
chrs=chr_list, gap_chr_data=gaps_chr_data, out_dump_file=out_folder + "input/hi-c_data/AAcol/norm_hic_data_" + str(chr_list) + ".pickle", obs_exp=False) logging.info(colored("succesfully normalize hi-c data", 'green')) # exit() # print(chr_norm_hic_data["X"]) # print(chr_norm_hic_data["X"].shape) # print("nonzeroes",np.count_nonzero(chr_norm_hic_data["X"])) #for interval in train or test intervals # get one-hot encoded sequence # get target hic matrix #write in test file logging.info( colored("going to generate encoded sequences and targets dataset", 'green')) genome = fastaReader(fasta_file, name="ACol") genome = genome.read_data() generate_train_dataset(seq_chr_data, genome, chr_norm_hic_data, chrms=set(chr_list), out_file=out_folder + "output/train_dataset_" + str(chr_list) + ".pickle", train_test="train", target_crop_bp=0) # generate_train_dataset(seq_chr_data, genome, chr_norm_hic_data, chrms=set(chr_list), out_file=out_folder+"output/test_dataset_"+str(chr_list)+".pickle", train_test="test", target_crop_bp=0)
fill_empty_contacts = False logging.getLogger(__name__).debug("Using input folder " + input_folder) # Read contacts data params.contacts_reader = ContactsReader() contacts_files = [input_folder + "19.contacts.gz"] coeff_fname = input_folder + "coefficient.NPC.5000.txt" # set path to the coefficient file and to contacts files # contacts file format: bin_start--bin_end--contact_count params.contacts_reader.read_files( contacts_files, coeff_fname, max_cpus=params.max_cpus, fill_empty_contacts=fill_empty_contacts, maxdist=params.maxdist) params.fastaReader = fastaReader( input_folder + "chr19.fa", chrm_names_renamer=rm_chr_from_chrName) params.fastaReader.read_data() print(params.fastaReader.data) SequencePG = SequencePredictorGenerator( fastaReader=params.fastaReader, binsize=params.contacts_reader.binsize) params.pgs = [SequencePG] params.out_file = output_folder + "NPC_5000" params.sample_size = 100 params.interval = Interval( "19", params.contacts_reader.get_min_contact_position("19"), params.contacts_reader.get_max_contact_position("19")) logging.getLogger(__name__).info("Generating dataset for interval " + str(params.interval)) generate_data(params)