def plot_hexbin_heatmap(xs, ys, xlabel, ylabel, xscale='log',
yscale='log', gridsize=100, colorscale=True, outfn=None):
'''
xscale: [ ‘linear’ | ‘log’ ]
Use a linear or log10 scale on the horizontal axis.
yscale: [ ‘linear’ | ‘log’ ]
Use a linear or log10 scale on the vertical axis.
gridsize: [ 100 | integer ]
The number of hexagons in the x-direction, default is 100. The
corresponding number of hexagons in the y-direction is chosen such that
the hexagons are approximately regular. Alternatively, gridsize can be
a tuple with two elements specifying the number of hexagons in the
x-direction and the y-direction.
'''
fig = plt.figure()
if colorscale:
plt.hexbin(xs, ys, bins='log', gridsize=gridsize, xscale=xscale,
yscale=yscale, mincnt=1, cmap=cm.get_cmap('jet'))
cb = plt.colorbar()
cb.set_label(r'$log_{10}(N)$', fontsize=16)
else:
plt.hexbin(xs, ys, gridsize=gridsize, xscale=xscale, yscale=yscale,
mincnt=1, cmap=cm.get_cmap('jet'))
cb = plt.colorbar()
cb.set_label('counts')
plt.xlabel(xlabel, linespacing=12, fontsize=18)
plt.ylabel(ylabel, linespacing=12, fontsize=18)
plt.tight_layout()
if outfn is not None:
fig.savefig(outfn)
plt.close()
# plt.show()
return fig
def hexlbin(a, b, c=None, **kwargs):
if c is None:
c = np.ones(len(a))
plt.hexbin(a,
b,
c,
reduce_C_function=lambda x: np.log(1 + np.sum(x)),
**kwargs)
def plot_hexbin(d, C, cmap, f):
plt.hexbin(np.array(d[f[0]]),
np.array(d[f[1]]),
C=C,
reduce_C_function=np.sum,
bins='log',
gridsize=200,
cmap=cmap)
plt.xlabel(f[0])
plt.ylabel(f[1])
plt.colorbar()
def colorbleed_plot(framea, frameb):
'''
Generates a plot which may help determine if there is colorbleed between two frames.
'''
import matplotlib.pylab as plt
plt.hexbin(framea.ravel(),
frameb.ravel(),
np.ones(np.prod(framea.shape)),
reduce_C_function=lambda x: np.log(np.sum(x)),
gridsize=50)
def mean_hexbin(XY, Z, s=57, gridsize=20, alpha=1.0, colormap='jet'):
hexdata = plt.hexbin(XY[:, 0], XY[:, 1], gridsize=gridsize)
plt.close()
plt.figure(figsize=(20, 13))
counts = hexdata.get_array()
verts = hexdata.get_offsets()
points = XY.reshape(XY.shape[0], -1)
binindex = np.argmin(
np.concatenate([((p - verts)**2).sum(1)[np.newaxis, :]
for p in points], 0), 1)
print(binindex.shape)
color = [
np.mean(np.array(Z)[binindex == i]) for i in xrange(verts.shape[0])
]
_ = plt.scatter(verts[:, 0],
verts[:, 1],
c=color,
s=s,
edgecolor='',
alpha=alpha,
marker='h',
cmap=colormap)
plt.colorbar(_)
def plotSkymap(data, name):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pylab as pl
from astropy.coordinates import SkyCoord
from astropy import units
coords = SkyCoord(ra=data['ra'],
dec=data['dec'],
unit='degree',
frame='icrs')
ra = coords.ra.wrap_at(180 * units.deg).radian
dec = coords.dec.radian
#pl.figure()
pl.subplot(111, projection="aitoff") # must use radians
#pl.subplot(111, projection="lambert")
# Make sure to bin the regions, otherwise Memory errors appear.
#pl.plot( ra , dec, 'ko', markersize=0.05 )
color_map = pl.cm.Spectral_r
image = pl.hexbin(ra,
dec,
cmap=color_map,
gridsize=100,
mincnt=1,
bins='log')
pl.colorbar(image, spacing='uniform', extend='max')
pl.xlabel('R.A')
pl.ylabel('DEC')
pl.legend()
pl.grid(True)
pl.savefig(name, dpi=150)
def plot_af_stats():
# Get histogram of valence and energy
plt.style.use('ggplot')
conn = sqlite3.connect(SPOTIFY_AF_DB)
cur = conn.cursor()
rows = cur.execute("SELECT valence, energy from SpotifyAudioFeatures")
valences, energies, valergies = [], [], []
for row in rows:
if row[0] and row[1]:
valences.append(row[0])
energies.append(row[1])
# if row[1]:
# if row[0] and row[1]:
# valergies.append([row[0], row[1]])
plt.hist(valences, 25)
plt.show()
plt.hist(energies, 25)
plt.show()
plt.hexbin(valences, energies, gridsize=25) #, bins='log', cmap='inferno')
plt.show()
def compute_all_stats():
""" Fetch all classification results and compute basic stats."""
h_x = []
h_label = []
### Fetch all results
# timestamp_prefix = "14"
# timestamp_prefix = "15[1,2]"
# timestamp_prefix = "153"
timestamp_prefix = "1"
for af, pfx_len in [(4, "24"), (6, "48")]:
all_classification_results = defaultdict(dict)
countNone = 0
all_files = glob.glob(esteban_results_directory +
"/{}*_{}".format(timestamp_prefix, pfx_len))
print("Reading {} input files...".format(len(all_files)))
for path in all_files:
dname = path.rpartition("/")[2]
ts, _, prefix = dname.partition("_")
prefix = prefix.replace("_", "/")
ts = int(ts)
# print("Processing %s %s..." % (ts, prefix))
asns = get_classification_results(ts, prefix)
if asns is not None and len(asns["zombie"]) > 200:
print("Large outbreak: {} {}".format(ts, prefix))
if asns is not None:
all_classification_results[ts][prefix] = asns
else:
countNone += 1
# if "2497" in asns["zombie"] :
# print("IIJ")
# print(ts,prefix,len(asns["zombie"]))
nb_outbreak = sum([
len(pfx_res.keys())
for ts, pfx_res in all_classification_results.items()
])
print("number of outbreaks: {}".format(nb_outbreak))
print("number of ts: {}".format(len(all_classification_results)))
print("number of nones: {}".format(countNone))
### AS hegemony
# if af == 4:
all_zombies = set([
asn for pfx_res in all_classification_results.values()
for res in pfx_res.values() for asn in res["zombie"]
])
hegemony = get_hegemony(all_zombies, af)
max_hege = []
outbreak_size = []
for ts, pfx_res in all_classification_results.items():
for pfx, res in pfx_res.items():
# for zombie in res["zombie"]:
hege = [hegemony[int(zombie)] for zombie in res["zombie"]]
max_hege.append(max(hege))
# max_hege.append(np.log(hegemony[int(zombie)]))
if max_hege[-1] == 0:
max_hege[-1] = min(hegemony.values())
print("max hege = 0! {}".format(res["zombie"]))
outbreak_size.append(len(res["zombie"]))
color_map = plt.cm.Spectral_r
plt.figure(100 + af)
heatmap = plt.hexbin(max_hege,
outbreak_size,
cmap=color_map,
mincnt=1,
bins="log",
gridsize=30,
xscale="log")
plt.xlabel("Max. AS Hegemony")
plt.ylabel("Outbreak size")
cb = plt.colorbar(heatmap, spacing='uniform', extend='max')
plt.tight_layout()
plt.savefig("fig/hege_outbreaksize_ipv{}.pdf".format(af))
### Temporal characteristics
ts_start = min(all_classification_results.keys())
ts_end = max(all_classification_results.keys())
# duration = (ts_end - ts_start)/3600 #hours
# nb_withdraws = duration/4
duration = 31 + 28 + 31 + 28 + 13 + 31 #days
nb_withdraws = duration * 6
print("First zombie detected at {} and last one at {}".format(
datetime.datetime.utcfromtimestamp(ts_start),
datetime.datetime.utcfromtimestamp(ts_end)))
nb_zombie_timebin = len(all_classification_results.keys())
perc_zombie_timebins = nb_zombie_timebin / nb_withdraws * 100
print(
"Percentage of withdraw periods with at least one zombie: {:.02f}%"
.format(perc_zombie_timebins))
print("Number of withdraw periods with zombies: {}".format(
len(all_classification_results)))
zombie_timebins_v4 = [
ts for ts, res in all_classification_results.items()
for pfx in res.keys() if "." in pfx
]
zombie_timebins_v6 = [
ts for ts, res in all_classification_results.items()
for pfx in res.keys() if ":" in pfx
]
perc_zombie_timebins_v4 = len(
set(zombie_timebins_v4)) / nb_withdraws * 100
perc_zombie_timebins_v6 = len(
set(zombie_timebins_v6)) / nb_withdraws * 100
print("Percentage of withdraw periods with one v4 zombie: {:.02f}%".
format(perc_zombie_timebins_v4))
print("Percentage of withdraw periods with one v6 zombie: {:.02f}%".
format(perc_zombie_timebins_v6))
zombies_per_timebin = [
set([asn for pfx, res in pfx_res.items() for asn in res["zombie"]])
for ts, pfx_res in all_classification_results.items()
]
normal_per_timebin = [
set([asn for pfx, res in pfx_res.items() for asn in res["normal"]])
for ts, pfx_res in all_classification_results.items()
]
nb_zombie_per_outbreak = [len(z) for z in zombies_per_timebin]
print("On average we have {:.02f} zombie AS per outbreak (median={})".
format(np.mean(nb_zombie_per_outbreak),
np.median(nb_zombie_per_outbreak)))
print("On average we have {:.02f} AS in the AS graph".format(
np.mean([
len(z.union(n))
for z, n in zip(zombies_per_timebin, normal_per_timebin)
])))
print("That's {:.02f}% zombie AS per outbreak".format(
np.mean([
100 * len(z) / (len(z) + len(n))
for z, n in zip(zombies_per_timebin, normal_per_timebin)
])))
plt.figure(1)
cdf = rfplt.ecdf(nb_zombie_per_outbreak, label="IPv{}".format(af))
plt.xlabel("Outbreak size")
plt.ylabel("CDF")
plt.legend(loc="best")
plt.tight_layout()
plt.savefig("fig/CDF_nb_zombie_per_outbreak.pdf")
print("CDF nb. ASes per outbreak:")
# print(cdf)
# Zombie Frequency for all beacons
asn_zombie_frequency = collections.Counter(
itertools.chain.from_iterable(zombies_per_timebin))
# add normal ASes:
for asn in set(itertools.chain.from_iterable(normal_per_timebin)):
if not asn in asn_zombie_frequency:
asn_zombie_frequency[asn] = 0
nb_outbreak = sum([
len(pfx_res.keys())
for ts, pfx_res in all_classification_results.items()
])
all_zombies = [[
asn for pfx, res in pfx_res.items() for asn in res["zombie"]
] for ts, pfx_res in all_classification_results.items()]
all_asn_zombie_frequency = collections.Counter(
itertools.chain.from_iterable(all_zombies))
print("Top zombie transit (nb. outbreak={}): ".format(nb_outbreak))
for asn, freq in all_asn_zombie_frequency.most_common(100):
if hegemony[int(asn)] > 0.001:
print("\t AS{}: {:.02f}% ({} times) hegemony={:.03f}".format(
asn, 100 * freq / nb_outbreak, freq, hegemony[int(asn)]))
plt.figure(2)
rfplt.ecdf(np.array(list(asn_zombie_frequency.values())) /
nb_zombie_timebin,
label="IPv{}".format(af))
plt.xlabel("freq. AS as zombie/total nb. of outbreaks")
plt.ylabel("CDF")
plt.legend(loc="best")
plt.tight_layout()
plt.savefig("fig/CDF_zombie_freq_per_asn.pdf")
# plt.show()
unique_pairs = [
set([
asn + "_" + pfx for pfx, res in pfx_res.items()
for asn in res["zombie"]
]) for ts, pfx_res in all_classification_results.items()
]
zombie_emergence = collections.Counter(
itertools.chain.from_iterable(unique_pairs))
plt.figure(22)
rfplt.ecdf(np.array(list(zombie_emergence.values())) / nb_withdraws,
label="IPv{}".format(af))
plt.xlabel("Zombie emergence rate")
plt.ylabel("CDF")
plt.legend(loc="best")
plt.tight_layout()
plt.savefig("fig/CDF_zombie_emergence_per_asn.pdf")
# plt.show()
print("Max. <ASN, beacon>: {} ({} times)".format(
max(zombie_emergence, key=zombie_emergence.get),
max(zombie_emergence.values())))
print("number of outbreaks: {}".format(nb_zombie_timebin))
for key, freq in zombie_emergence.items():
if freq > 590:
print("High. <ASN, beacon>: {} ({} times)".format(key, freq))
### Zombie frequency per beacon
all_beacons = set([
pfx for pfx_res in all_classification_results.values()
for pfx in pfx_res.keys()
])
plt.figure(3)
print(all_beacons)
for pfx in all_beacons:
zombies_per_timebin_per_beacon = [
set([asn for asn in pfx_res[pfx]["zombie"]])
for ts, pfx_res in all_classification_results.items()
if pfx in pfx_res
]
asn_zombie_frequency = collections.Counter(
itertools.chain.from_iterable(zombies_per_timebin_per_beacon))
# add normal ASes:
for asn in set(itertools.chain.from_iterable(normal_per_timebin)):
if not asn in asn_zombie_frequency:
asn_zombie_frequency[asn] = 0
# print("Top 10 zombie ASN for {}: ".format(pfx))
# for asn, freq in asn_zombie_frequency.most_common(10):
# print("\t AS{}: {:.02f}% ({} times)".format(asn, 100*freq/len(zombies_per_timebin_per_beacon), freq))
rfplt.ecdf(np.array(list(asn_zombie_frequency.values())) /
len(zombies_per_timebin_per_beacon),
label=pfx)
plt.xlabel("freq. AS as zombie/total nb. of outbreaks")
plt.ylabel("CDF")
plt.legend(loc="best", prop={"size": 8}, ncol=2)
plt.tight_layout()
plt.savefig("fig/CDF_zombie_freq_per_asn_per_beacon.pdf")
# plt.show()
# from efficient_apriori import apriori
# itemsets, rules = apriori(zombies_per_timebin, min_support=0.3, min_confidence=1)
# print(zombies_per_timebin)
# for nb_elem, items in itemsets.items():
# if nb_elem > 1:
# print(nb_elem)
# print(items)
# rules_rhs = filter(lambda rule: len(rule.lhs)==1 and len(rule.rhs)>4, rules)
# print(list(rules_rhs))
nb_outbreak_per_prefix = defaultdict(int)
for ts, events in all_classification_results.items():
for prefix, classification in events.items():
nb_outbreak_per_prefix[prefix] += 1
# print(nb_outbreak_per_prefix)
h_values = list(nb_outbreak_per_prefix.values())
plt.figure()
h_x.append(h_values)
h_label.append("IPv{}".format(af))
plt.hist(h_x, label=h_label)
plt.xlabel("Number of outbreaks per beacon")
plt.ylabel("Number of beacons")
plt.legend(loc="best")
plt.tight_layout()
plt.savefig("fig/hist_nb_outbreak_per_prefix.pdf")
print("On average {} outbreaks per beacon (max={})".format(
np.mean(h_values), np.max(h_values)))
print("Number of oubreaks per beacon: ", nb_outbreak_per_prefix)
nb_beacon_per_ts = {
ts: len(events)
for ts, events in all_classification_results.items()
}
# print(nb_beacon_per_ts)
plt.figure(5)
cdf = rfplt.ecdf(list(nb_beacon_per_ts.values()),
label="IPv{}".format(af))
plt.xlabel("Number of simultaneous outbreaks")
plt.ylabel("CDF")
plt.legend(loc="best")
plt.xlim([0.5, 14.5])
plt.tight_layout()
plt.savefig("fig/CDF_nb_simult_zombie.pdf")
# print(cdf)
print(max(nb_beacon_per_ts, key=nb_beacon_per_ts.get))
qst = seb_dat[:, 24 - 1] qm = seb_dat[:, 25 - 1] # calculate the temperature dependent flux t_dep_flux = lw_net + qs + ql + qc + qst # calculate the melt energy - sw_net and precip energy (what the ETI TF accounts for) qm_wo_sw_prc = qm - sw_net - qprc # reset to 0 when no melt, as SWnet + qm_wo_sw_prc should never be negative. qm_wo_sw_prc[(qm == 0)] = 0 ta = seb_dat[:, 8 - 1] ea = seb_dat[:, 10 - 1] ws = seb_dat[:, 7 - 1] # plot for all melting periods plt.figure() plt.hexbin(ta[~(qm_wo_sw_prc == 0)], qm_wo_sw_prc[~(qm_wo_sw_prc == 0)], cmap=plt.cm.inferno_r) # all periods with melt a2 = linregress(ta[~(qm_wo_sw_prc == 0)], qm_wo_sw_prc[~(qm_wo_sw_prc == 0)]) plt.plot(np.linspace(-5, 12.5, 100), np.linspace(-5, 12.5, 100) * a2[0] - a2[1], 'b') a = curve_fit(f, ta[~(qm_wo_sw_prc == 0)], qm_wo_sw_prc[~(qm_wo_sw_prc == 0)]) plt.plot(np.linspace(-5, 12.5, 100), np.linspace(-5, 12.5, 100) * a[0][0] + a[1][0], 'b--') # all periods b2 = linregress(ta, qm_wo_sw_prc) plt.plot(np.linspace(-5, 12.5, 100), np.linspace(-5, 12.5, 100) * b2[0] - b2[1], 'r') b = curve_fit(f, ta, qm_wo_sw_prc) plt.plot(np.linspace(-5, 12.5, 100), np.linspace(-5, 12.5, 100) * b[0][0] - b[1][0], 'r--') # all periods with positive melt energy from Ta c2 = linregress(ta[(qm_wo_sw_prc > 0)], qm_wo_sw_prc[(qm_wo_sw_prc > 0)])
sig = signals.ResampledQAM(M, N, nmodes=2, fb=fb, fs=fs, resamplekwargs={"beta":0.01, "renormalise":True})
sig = impairments.change_snr(sig, snr)
SS = impairments.apply_PMD(sig, theta, t_pmd)
E_s, wxy_s, (err_s, err_rde_s) = equalisation.dual_mode_equalisation(SS, (muCMA, muRDE), ntaps,
methods=("mcma", "mrde"))
E_s = helpers.normalise_and_center(E_s)
evm = sig[:, ::2].cal_evm()
evmE_s = E_s.cal_evm()
gmiE = E_s.cal_gmi()
print(gmiE)
plt.figure()
plt.subplot(221)
plt.hexbin(E_s[0].real, E_s[0].imag)
plt.text(0.999, 0.9, r"$EVM_x={:.1f}\%$".format(100*evmE_s[0]), color='w', horizontalalignment="right", fontsize=14)
plt.subplot(222)
plt.title('Recovered MCMA/MRDE')
plt.hexbin(E_s[1].real, E_s[1].imag)
plt.text(0.999, 0.9, r"$EVM_y={:.1f}\%$".format(100*evmE_s[1]), color='w', horizontalalignment="right", fontsize=14)
plt.subplot(223)
plt.title('Original')
plt.hexbin(sig[0,::2].real, sig[0,::2].imag)
plt.text(0.999, 0.9, r"$EVM_x={:.1f}\%$".format(100*evm[0]), color='w', horizontalalignment="right", fontsize=14)
plt.subplot(224)
plt.hexbin(sig[1,::2].real, sig[1,::2].imag)
plt.text(0.999, 0.9, r"$EVM_y={:.1f}\%$".format(100*evm[1]), color='w', horizontalalignment="right", fontsize=14)
plt.figure()
plt.subplot(221)
print(
np.sum(ta > 0),
np.sum(np.logical_and(ta > 0, qm_wo_sw_prc > 0)),
np.sum(qm_wo_sw_prc > 0),
np.sum(np.logical_and(ta > 0, qm_wo_sw_prc > 0)) / np.sum(ta > 0),
)
print(
np.sum(ea > 6.112),
np.sum(np.logical_and(ea > 6.1120, qm_wo_sw_prc > 0)),
np.sum(qm_wo_sw_prc > 0),
np.sum(np.logical_and(ea > 6.1120, qm_wo_sw_prc > 0)) / np.sum(ea > 6.112),
)
plt.figure()
plt.hexbin(qm_wo_sw_prc[(qm_wo_sw_prc > 0)],
ta[(qm_wo_sw_prc > 0)],
cmap=plt.cm.inferno_r)
plt.plot(range(200), np.arange(200) / 14.7, 'k')
plt.plot(range(100), np.arange(100) / 8.7, 'r')
plt.xlabel('QM - SWnet - Qprecip')
plt.ylabel('Air temperature (C)')
plt.savefig(r'D:\Snow project\Oct2018 Results\qm_wo_sw_prc vs ta posQM.png')
plt.figure()
plt.hexbin(qm_wo_sw_prc[(qm_wo_sw_prc > 0)],
ea[(qm_wo_sw_prc > 0)],
cmap=plt.cm.inferno_r)
plt.plot(range(200), 6.112 + np.arange(200) / 42.0, 'k')
plt.xlabel('QM - SWnet - Qprecip')
plt.ylabel('Vapour pressure (hPa)')
plt.savefig(r'D:\Snow project\Oct2018 Results\qm_wo_sw_prc vs ea posQM.png')
#sig4 = impairments.apply_PMD(sig4, theta, dgd)
#sig4 = impairments.apply_PMD(sig4, theta, dgd)
#sig4[0,:] = sig3[1, 20000:]
#sig4[1,:] = sig3[0, 20000:]
sig4.sync2frame(Ntaps=ntaps)
wx, s1 = equalisation.pilot_equaliser(sig4, [1e-3, 1e-3],
ntaps,
True,
adaptive_stepsize=True)
s2, ph = phaserec.pilot_cpe(s1, nframes=1)
gmi = s2.cal_gmi()
evm = s2.cal_evm()
ber = s2.cal_ber()
ser = s2.cal_ser()
ksnr = s2.est_snr()
print("gmi {}".format(gmi))
print("ber {}".format(ber))
print("evm {}".format(evm))
print("snr {}".format(snr))
print("ser {}".format(ser))
plt.figure()
plt.subplot(121)
plt.title("Without CPE")
plt.hexbin(s1[0].real, s1[0].imag)
plt.subplot(122)
plt.title("With CPE")
plt.hexbin(s2[0].real, s2[0].imag)
plt.show()
sn = signals.SignalWithPilots.from_data_array(sig.symbols[0].reshape(1, -1), )
plt.subplot(133)
plt.title('Original')
plt.plot(S[0, ::2].real,
S[0, ::2].imag,
'ro',
label=r"$SER_x=%.1f\%%$" % (100 * ser0[0]))
plt.plot(S[1, ::2].real,
S[1, ::2].imag,
'go',
label=r"$SER_y=%.1f\%%$" % (100 * ser0[1]))
plt.legend()
plt.figure()
plt.subplot(221)
plt.title('Recovered CMA X')
plt.hexbin(E[0].real, E[0].imag)
plt.text(0.999,
0.9,
r"$GMI_x={:.1f}$".format(gmi[0]),
color='w',
horizontalalignment="right",
fontsize=14)
plt.subplot(222)
plt.title('Recovered CMA Y')
plt.hexbin(E[1].real, E[1].imag)
plt.text(0.999,
0.9,
r"$GMI_y={:.1f}$".format(gmi[1]),
color='w',
horizontalalignment="right",
fontsize=14)
adaptive_stepsize=(True, True))
E = helpers.normalise_and_center(E)
E_s = helpers.normalise_and_center(E_s)
E_m = helpers.normalise_and_center(E_m)
evm = sig[:, ::2].cal_evm()
evmE = E.cal_evm()
evmE_m = E_m.cal_evm()
evmE_s = E_s.cal_evm()
gmiE = E.cal_gmi()
print(gmiE)
plt.figure()
plt.subplot(241)
plt.title('Recovered MCMA/MDDMA')
plt.hexbin(E[0].real, E[0].imag, label=r"$EVM_x=%.1f\%%$" % (evmE[0] * 100))
plt.legend()
plt.subplot(242)
plt.hexbin(E[1].real, E[1].imag, label=r"$EVM_y=%.1f\%%$" % (100 * evmE[1]))
plt.legend()
plt.subplot(243)
plt.title('Recovered MCMA/MRDE')
plt.hexbin(E_m[0].real,
E_m[0].imag,
label=r"$EVM_x=%.1f\%%$" % (evmE_m[0] * 100))
plt.legend()
plt.subplot(244)
plt.hexbin(E_m[1].real,
E_m[1].imag,
label=r"$EVM_y=%.1f\%%$" % (100 * evmE_m[1]))
plt.legend()
def main():
savePath = '/Users/umut/Projects/intragenicTranscription/analysis/'
gdb = genome.db.GenomeDB(
path='/Users/umut/Projects/genome/data/share/genome_db',
assembly='sacCer3')
with open('../analysis/TSSseq_replicates.txt', 'r') as f:
datNames = [t.split('\n')[0].split('\t') for t in f.readlines()]
dataDict = {}
for datNam1, datNam2 in datNames:
dataDict[datNam1 + '_pos'] = gdb.open_track(datNam1 + '_pos')
dataDict[datNam1 + '_neg'] = gdb.open_track(datNam1 + '_neg')
dataDict[datNam2 + '_pos'] = gdb.open_track(datNam2 + '_pos')
dataDict[datNam2 + '_neg'] = gdb.open_track(datNam2 + '_neg')
chname = 'chrIV'
pos = 0
width = 1500000
xran = np.array([1, 2, 5, 10, 15, 20, 100])
logcorcoef = {}
corcoef = {}
for smps in tqdm(datNames):
smp1 = smps[0]
smp2 = smps[1]
pdf = PdfPages(savePath + smp1 + '_' + smp2 + '_jointplots.pdf')
smp1reads=np.r_[dataDict[smp1+'_pos'].get_nparray(chname,pos+1,(pos+width)),\
dataDict[smp1+'_neg'].get_nparray(chname,pos+1,(pos+width))]
smp2reads=np.r_[dataDict[smp2+'_pos'].get_nparray(chname,pos+1,(pos+width)),\
dataDict[smp2+'_neg'].get_nparray(chname,pos+1,(pos+width))]
qq = 0
logcorcoef[smp1 + '_' + smp2] = np.zeros((len(xran)))
corcoef[smp1 + '_' + smp2] = np.zeros((len(xran)))
for window_size in tqdm(xran):
if window_size > 1:
smp1reads = running_mean(smp1reads, window_size)
smp2reads = running_mean(smp2reads, window_size)
logcorcoef[smp1 + '_' + smp2][qq] = np.corrcoef(
np.log10(smp1reads + 1e-1), np.log10(smp2reads + 1e-1))[1][0]
corcoef[smp1 + '_' + smp2][qq] = np.corrcoef(smp1reads,
smp2reads)[1][0]
pl.figure()
pl.hexbin(np.log10(smp1reads + 1),
np.log10(smp2reads + 1),
bins='log',
cmap=pl.cm.YlOrRd_r)
pl.xlabel(smp1)
pl.ylabel(smp2)
pl.title('pearson log-corr: ' +
str(logcorcoef[smp1 + '_' + smp2][qq]) + 'window ' +
str(window_size))
pdf.savefig()
pl.close()
pl.figure()
pl.hexbin(smp1reads, smp2reads, bins='log', cmap=pl.cm.YlOrRd_r)
pl.xlabel(smp1)
pl.ylabel(smp2)
pl.title('pearson corr: ' + str(corcoef[smp1 + '_' + smp2][qq]) +
'window ' + str(window_size))
pdf.savefig()
pl.close()
qq += 1
pl.figure()
pl.plot(np.log10(xran), logcorcoef[smp1 + '_' + smp2])
pl.xlabel('log10 window size')
pl.ylabel('pearson log-correlation score')
pl.title(smp1 + ' vs. ' + smp2)
pdf.savefig()
pl.close()
pl.figure()
pl.plot(np.log10(xran), corcoef[smp1 + '_' + smp2])
pl.xlabel('log10 window size')
pl.ylabel('pearson correlation score')
pl.title(smp1 + ' vs. ' + smp2)
pdf.savefig()
pl.close()
pdf.close()
for datNam1, datNam2 in datNames:
dataDict[datNam1 + '_pos'].close()
dataDict[datNam1 + '_neg'].close()
dataDict[datNam2 + '_pos'].close()
dataDict[datNam2 + '_neg'].close()
pdf = PdfPages(savePath + 'All_corrScores.pdf')
pl.figure()
for smps in datNames:
smp1 = smps[0]
smp2 = smps[1]
pl.plot(np.log10(xran),
logcorcoef[smp1 + '_' + smp2],
label=smp1 + '_' + smp2)
pl.legend()
pl.xlabel('log10 window size')
pl.ylabel('pearson log-correlation score')
pl.title('All')
pdf.savefig()
pl.close()
pl.figure()
for smps in datNames:
smp1 = smps[0]
smp2 = smps[1]
pl.plot(np.log10(xran),
corcoef[smp1 + '_' + smp2],
label=smp1 + '_' + smp2)
pl.legend()
pl.xlabel('log10 window size')
pl.ylabel('pearson correlation score')
pl.title('All')
pdf.savefig()
pl.close()
pdf.close()
print("Y pol phase")
Ey, phy = phaserecovery.bps(E[1], 32, QAM.symbols, 8)
print("Y pol phase done")
Ec = np.vstack([Ex,Ey])
evmX = QAM.cal_evm(X[::2])
evmY = QAM.cal_evm(Y[::2])
evmEx = QAM.cal_evm(E[0])
evmEy = QAM.cal_evm(E[1])
#Ec[0] = signal_quality.normalise_sig(Ec[0], M)[1]
#Ec[1] = signal_quality.normalise_sig(Ec[1], M)[1]
evmEx_c = QAM.cal_evm(Ec[0])
evmEy_c = QAM.cal_evm(Ec[1])
print(evmEy_c)
print(evmEx_c)
#sys.exit()
plt.figure()
plt.subplot(221)
#plt.title(r'After Phaserecovery $EVM_x=%.1f\%%$'%(evmEx_c*100))
plt.hexbin(Ec[0].real, Ec[0].imag)
plt.subplot(222)
#plt.title(r'After Phaserecovery $EVM_y=%.1f\%%$'%(evmEy_c*100))
plt.hexbin(Ec[1].real, Ec[1].imag)
plt.subplot(223)
#plt.title(r'Before Phaserecovery $EVM_x=%.1f\%%$'%(evmEx*100))
plt.hexbin(E[0].real, E[0].imag)
plt.subplot(224)
#plt.title(r'Before Phaserecovery $EVM_y=%.1f\%%$'%(evmEy*100))
plt.hexbin(E[1].real, E[1].imag)
plt.show()
histtype='stepfilled',
color="blue",
range=(min_distance, max_distance),
label="contacts per 1 A")
plt.hist(cont,
bins=(max_distance - min_distance) * 2,
histtype='stepfilled',
color="red",
alpha=0.5,
label="contacts per 0.5 A",
range=(min_distance, max_distance))
plt.xlabel('distance between atoms')
plt.ylabel('number of contacts')
plt.subplot(122, axisbg='darkblue')
gridsize = lambda f: max(f) if max(f) >= 100 else 100
plt.hexbin(numseqA,
numseqB,
z,
gridsize=gridsize(numseqA),
cmap=plt.cm.jet_r,
vmin=min_distance,
vmax=max_distance)
plt.axis([0, max(numseqA), 0, max(numseqB)])
plt.xlabel('sequence')
plt.ylabel('sequence')
plt.title("Contacts map")
cb = plt.colorbar()
cb.set_label('distance, A')
plt.show()
def hexbin(x, y, color, **kwargs):
cmap = sns.light_palette(color, as_cmap=True)
plt.hexbin(x, y, gridsize=25, cmap=cmap, **kwargs)
