def test_to_numeric(self):
scales = cl.to_numeric(cl.scales['3']['div']['RdYlBu'])
self.assertEqual(
scales,
[(252, 141, 89),
(255, 255, 191), (145, 191, 219)]
)
def __plot_tsne_to_visdom(z_embed, classes):
import colorlover as cl
C = np.array([
list(x) for x in cl.to_numeric(cl.scales['10']['qual']['Paired'])
]).astype(int)
for ic in range(10):
idx = classes[:, ic] == 1
X = z_embed[idx, :]
Y = np.ones_like(X[:, 0]).astype(int) # treat as a single class
Ci = np.expand_dims(C[ic], axis=0) # pickup a corresponding color
viz_plot(f"z_tsne_for_{ic}",
viz.scatter,
X,
Y,
opts=dict(markercolor=Ci, markersize=4, legend=[str(ic)]))
X = z_embed
Y = np.argmax(classes, axis=1) + 1
viz_plot("z_tsne",
viz.scatter,
X,
Y,
opts=dict(width=800,
height=800,
markercolor=C,
markersize=4,
legend=[str(x) for x in range(10)]))
def reset_world(self, world):
# random properties for agents
world.goals = []
n = 10
colors = np.array(cl.to_numeric(
cl.scales[str(10)]['div']['RdYlBu'])) / 255
for threat in world.threats:
self.generate_random_threat(threat, world)
for i, agent in enumerate(world.agents):
agent.size = np.random.uniform(0.2, 0.3)
agent.pseudo_collision_range = agent.size + 0.1
agent.color = colors[i % n]
agent.target.color = colors[i % n]
# agent.state.p_pos = np.random.uniform(-1, +1, world.dim_p)
if i % 2 == 0:
self.generate_random_pose(agent, world)
self.generate_random_goal(agent, world)
else:
agent.state.p_pos = np.copy(world.agents[i -
1].target.state.p_pos)
agent.target.state.p_pos = np.copy(world.agents[i -
1].state.p_pos)
world.goals.append(np.copy(agent.target.state.p_pos))
agent.state.p_vel = np.zeros(world.dim_p)
agent.previous_state.p_pos = np.copy(agent.state.p_pos)
agent.previous_state.p_vel = np.copy(agent.state.p_vel)
agent.state.c = np.zeros(world.dim_c)
agent.terminate = False
for agent in world.agents:
agent.agents_lidar = world.lidar.get_ray_lidar(agent)
agent.lidar_memory = [agent.agents_lidar, agent.agents_lidar]
def generate_cluster_colors(num):
"""Generate a list of colors given number needed.
Arguments:
num (int): Number of colors needed. n <= 35.
Returns:
list: strings containing RGB-style strings e.g. rgb(255,255,255).
"""
# Selects a random colorscale (RGB) depending on number of colors needed
if num < 12:
c = cl.scales[str(num)]['qual']
c = c[random.choice(list(c))]
else:
num_rounded = int(math.ceil(num / 10)) * 10
c = cl.to_rgb(cl.interp(cl.scales['12']['qual']['Paired'],
num_rounded))
c = cl.to_numeric(sample(c, int(num)))
# Converts selected colorscale to HSL, darkens the color if it is too light,
# convert it to rgb string and return
c_rgb = []
for color in c:
hls = colorsys.rgb_to_hls(color[0] / 255, color[1] / 255,
color[2] / 255)
if hls[1] < 0.6: # Darkens the color if it is too light (HLS = [0]Hue [1]Lightness [2]Saturation)
rgb = colorsys.hls_to_rgb(hls[0], 0.6, hls[2])
else:
rgb = colorsys.hls_to_rgb(hls[0], hls[1], hls[2])
rgb_str = "rgb(" + str(rgb[0] * 255) + "," + str(
rgb[1] * 255) + "," + str(rgb[2] * 255) + ")"
c_rgb.append(rgb_str)
return c_rgb
def test_to_numeric(self):
scales = cl.to_numeric(cl.scales['3']['div']['RdYlBu'])
self.assertEqual(
scales,
[(252, 141, 89),
(255, 255, 191), (145, 191, 219)]
)
def custom_color(sample_to_label, samples_dendro_leaves, color_scale='Spectral'):
# Sample label fragmentation
sample_to_label = sample_to_label.astype(str).fillna('unkown')
label_leaves = [str(sample_to_label[x]) for x in samples_dendro_leaves]
uni_labels = list(np.unique(sample_to_label))
uni_len = len(uni_labels)
uni_range = range(uni_len)
uni_normal = [x/uni_range[-1] for x in uni_range]
val_label = pd.Series(uni_range, index=uni_labels)
val_set = [val_label[x] for x in label_leaves]
colors = cl.scales[str(min(max(uni_len, 3), 11))]['div']['Spectral']
if uni_len > len(colors):
if uni_len > 20:
colors = cl.to_rgb(cl.interp(colors, 20))
for i in range(uni_len - 20):
i = i % 20
colors.append(colors[i])
else:
colors = cl.to_rgb(cl.interp(colors, uni_len))
colors_legend = [tuple(map(lambda x:x/255, x)) for x in cl.to_numeric(colors)]
c_map_legend = list(zip(uni_range, colors_legend))
c_map = list(zip(uni_normal, colors))
return c_map, c_map_legend, val_set, val_label
def get_continuous_cmap(hex_list, float_list=None, n_bins=256, rgb=False):
''' creates and returns a color map that can be used in heat map figures.
If float_list is not provided, colour map graduates linearly between each color in hex_list.
If float_list is provided, each color in hex_list is mapped to the respective location in float_list.
Parameters
----------
hex_list: list of hex code strings
float_list: list of floats between 0 and 1, same length as hex_list. Must start with 0 and end with 1.
rgb_list: bool indiciating that input is already in rgb
Returns
----------
colour map'''
if not rgb:
rgb_list = [rgb_to_dec(hex_to_rgb(i)) for i in hex_list]
else:
hex_list = colorlover.to_numeric(hex_list)
rgb_list = [rgb_to_dec(i) for i in hex_list]
if float_list:
pass
else:
float_list = list(np.linspace(0, 1, len(rgb_list)))
cdict = dict()
for num, col in enumerate(['red', 'green', 'blue']):
col_list = [[float_list[i], rgb_list[i][num], rgb_list[i][num]] for i in range(len(float_list))]
cdict[col] = col_list
cmp = colors.LinearSegmentedColormap('my_cmp', segmentdata=cdict, N=n_bins)
return cmp
def get_plot_colors(max_colors, color_format="pyplot"):
"""Generate colors for plotting."""
colors = cl.scales["11"]["qual"]["Paired"]
if max_colors > len(colors):
colors = cl.to_rgb(cl.interp(colors, max_colors))
if color_format == "pyplot":
return [[j / 255.0 for j in c] for c in cl.to_numeric(colors)]
return colors
def get_plot_colors(max_colors, color_format='pyplot'):
"""Generate colors for plotting."""
colors = cl.scales['11']['qual']['Paired']
if max_colors > len(colors):
colors = cl.to_rgb(cl.interp(colors, max_colors))
if color_format == 'pyplot':
return [[j / 255.0 for j in c] for c in cl.to_numeric(colors)]
return colors
def to_color(self, value):
if self._max == self._min:
return "#fffffff"
pct = (value - self._min) / (self._max - self._min)
bin_no = max(min(int(pct * self._n), self._n - 1), 0)
color = self._scheme[bin_no]
r, g, b = colorlover.to_numeric(colorlover.to_rgb([color]))[0]
hexval = "#{0:02x}{1:02x}{2:02x}".format(int(r), int(g), int(b))
return hexval
def to_hex(scale):
"""
Convert an hsl, numeric or rgb color to string hex color. ie,
[ "hsl(360,100,100)", "hsl(360,100,100)", "hsl(360,100,100)" ] -->
[ "#FFFFFF", "#FFFFFF", "#FFFFFF" ]
"""
s_t = cl.scale_type(scale)
if s_t == 'hex':
return scale
elif s_t == 'numeric':
return [
'#%02x%02x%02x' % tuple(map(int, s)) for s in cl.to_numeric(scale)
]
elif s_t == 'rgb':
return [
'#%02x%02x%02x' % tuple(map(int, s)) for s in cl.to_numeric(scale)
]
elif s_t == 'hsl':
return [
'#%02x%02x%02x' % tuple(map(int, s)) for s in cl.to_numeric(scale)
]
def get_hex_colors(colors="Greys"):
""" Creates an array of hex coded colors.
:param colors: string color group, Available colors can be found on # https://plot.ly/ipython-notebooks/color-scales/.
:return: array
"""
hex_colors = []
num_colors = cl.to_numeric(cl.scales['9']['seq'][colors])
for num_color in num_colors:
num_color = [int(x) for x in num_color]
num_color = '#%02x%02x%02x' % tuple(num_color)
hex_colors.append(num_color)
return hex_colors
def color_scale_interp(
input_num,
max_num,
min_num,
color_mesh_size=80,
hex_mode=True,
):
"""
"""
# | - color_scale_interp
# cl.scales["8"]["seq"]["Purples"]
black_white_cs = [
'rgb(0,0,0)',
'rgb(255,255,255)',
]
black_red_cs = [
'rgb(0,0,0)',
'rgb(255,0,0)',
]
color_scale_i = black_red_cs
color_scale_i = cl.scales["8"]["seq"]["Greys"][::-1]
# color_scale_i = cl.scales["8"]["seq"]["Purples"]
# color_scale_i = cl.scales['3']['div']['RdYlBu']
color_scale = cl.interp(
color_scale_i,
color_mesh_size,
)
color_scale = cl.to_rgb(color_scale)
# Converting RGB string representatino to tuple
color_scale = cl.to_numeric(color_scale)
input_norm = ((input_num - min_num) / (max_num - min_num))
cs_index = round(input_norm * len(color_scale)) - 1
if cs_index == -1:
cs_index = 0
color_out = color_scale[cs_index]
if hex_mode:
color_out = rgb_to_hex(color_out)
return (color_out)
def getpaintedimage(image: np.ndarray, masks: np.ndarray) -> np.ndarray:
'''Paints mask on image
Masks are painted in different colours automaticly.
Parameters
----------
image: numpy.ndarray
The image to be painted. Must be numpy.ndarray of shape (w,h,c) and
type np.uint8.
masks: numpy.ndarray
The mask for painting. Must be numpy.ndarray of shape (w,h,c) and
type np.uint8.
Returns
-------
numpy.ndarray
A array representing the painted image in format (w,h,c) of
type np.uint8
'''
result = image.copy()
masklist = tuple(masks[:, :, i] for i in range(masks.shape[2]))
num_classes = masks.shape[2]
colors = cl.scales[f"{num_classes}"]['qual']['Set3']
labels = np.array(range(1, num_classes + 1))
palette = dict(zip(labels, np.array(cl.to_numeric(colors))))
for i in range(num_classes):
mask_layer = masklist[i]
color = palette[i + 1]
contours, _ = cv2.findContours(mask_layer, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
result = cv2.drawContours(
image=result,
contours=contours,
contourIdx=-1,
color=color,
thickness=2)
if mask_layer.sum() > 0:
width = image.shape[0]
scale = 0.4
cv2.putText(
img=result,
text=f"{i+1}",
org=(int(20 * scale) + (i * 16), int(40 * scale)),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=scale,
color=color,
thickness=1,
lineType=cv2.LINE_AA)
return result
def text_contrast_color(bg_color):
""" Decides if text should be black or white based on background luminance
:param bg_color: RGB color
:return:
"""
bg_color_numeric = colorlover.to_numeric([bg_color])[0]
# Computing luminance
p_luminance = (0.299 * bg_color_numeric[0] + 0.587 * bg_color_numeric[1] + 0.114 * bg_color_numeric[2]) / 255;
if (p_luminance > 0.6):
text_color = 'black'
else:
text_color = 'white'
return text_color
def reset_world(self, world):
colors = np.array(cl.to_numeric(cl.scales['5']['div']['RdYlBu'])) / 255
for i, agent in enumerate(world.agents):
agent.size = np.random.uniform(0.2, 0.3)
agent.pseudo_collision_range = agent.size + 0.1
agent.color = colors[i % 5]
agent.target.color = colors[i % 5]
self.generate_random_pose(agent)
self.generate_random_goal(agent)
agent.state.p_vel = np.zeros(world.dim_p)
agent.previous_state.p_pos = np.copy(agent.state.p_pos)
agent.previous_state.p_vel = np.copy(agent.state.p_vel)
agent.state.c = np.zeros(world.dim_c)
agent.terminate = False
for agent in world.agents:
agent.agents_lidar = world.lidar.get_ray_lidar(agent)
agent.lidar_memory = [agent.agents_lidar, agent.agents_lidar]
def _set_progress(self, progress):
clear()
colors = cl.to_numeric(cl.scales['8']['div']['RdYlGn'])
ledNum = int(round((progress / 100) * 7))
for i in range(0, ledNum):
self._blinkt_r = int(colors[ledNum][0])
self._blinkt_g = int(colors[ledNum][1])
self._blinkt_b = int(colors[ledNum][2])
self._logger.info("Setting " + str(i) + " to " +
str(self._blinkt_r) + "," + str(self._blinkt_g) +
"," + str(self._blinkt_b))
set_pixel(i, self._blinkt_r, self._blinkt_g, self._blinkt_b, 0.5)
show()
def __init__(self, colors):
if isinstance(colors, (LinearSegmentedColormap, ListedColormap)):
colors = colors
else:
if not isinstance(colors[0], str):
# Consider it an array of rgb/rgba
colors = np.atleast_2d(colors).astype(float)
if colors.shape[-1] not in [3, 4]:
raise ValueError('if floats/ints, colors must have'
' a last dimension of shape 3 or 4')
if not ((colors >= 0) * (colors <= 1)).all():
colors = colors / 255.
else:
if 'rgb' in colors[0]:
# Convert strings to numeric first
colors = np.array(cl.to_numeric(colors)) / 255.
elif '#' in colors[0]:
# Assume it's hex and we can just pass
pass
else:
# Try to convert with webcolors
colors = _names_to_rgb(colors)
colors = pl.blend_palette(colors, as_cmap=True)
self.cmap = colors
def interp(scl, r):
'''
Replacement for colorlover.interp
Interpolate a color scale "scl" to a new one with length "r"
Fun usage in IPython notebook:
HTML( to_html( to_hsl( interp( cl.scales['11']['qual']['Paired'], 5000 ) ) ) )
'''
c = []
SCL_FI = len(scl) - 1 # final index of color scale
# garyfeng:
# the following line is buggy.
# r = [x * 0.1 for x in range(r)] if isinstance( r, int ) else r
r = [x * 1.0 * SCL_FI / r for x in range(r)] if isinstance(r, int) else r
# end garyfeng
scl = cl.to_numeric(scl)
def interp3(fraction, start, end):
'''Interpolate between values of 2, 3-member tuples'''
def intp(f, s, e):
return s + (e - s) * f
return tuple([intp(fraction, start[i], end[i]) for i in range(3)])
def rgb_to_hsl(rgb):
'''
Adapted from M Bostock's RGB to HSL converter in d3.js
https://github.com/mbostock/d3/blob/master/src/color/rgb.js
'''
r, g, b = float(rgb[0]) / 255.0,\
float(rgb[1]) / 255.0,\
float(rgb[2]) / 255.0
mx = max(r, g, b)
mn = min(r, g, b)
h = s = l = (mx + mn) / 2
if mx == mn: # achromatic
h = 0
s = 0 if l > 0 and l < 1 else h
else:
d = mx - mn
s = d / (mx + mn) if l < 0.5 else d / (2 - mx - mn)
if mx == r:
h = (g - b) / d + (6 if g < b else 0)
elif mx == g:
h = (b - r) / d + 2
else:
h = r - g / d + 4
return (int(round(h * 60, 4)), int(round(s * 100,
4)), int(round(l * 100, 4)))
for i in r:
# garyfeng: c_i could be rounded up so scl[c_i+1] will go off range
# c_i = int(i*math.floor(SCL_FI)/round(r[-1])) # start color index
# c_i = int(math.floor(i*math.floor(SCL_FI)/round(r[-1]))) # start color index
# c_i = if c_i < len(scl)-1 else hsl_o
c_i = int(math.floor(i))
section_min = math.floor(i)
section_max = math.ceil(i)
fraction = (i - section_min) # /(section_max-section_min)
hsl_o = rgb_to_hsl(scl[c_i]) # convert rgb to hls
hsl_f = rgb_to_hsl(scl[c_i + 1])
# section_min = c_i*r[-1]/SCL_FI
# section_max = (c_i+1)*(r[-1]/SCL_FI)
# fraction = (i-section_min)/(section_max-section_min)
hsl = interp3(fraction, hsl_o, hsl_f)
c.append('hsl' + str(hsl))
return cl.to_hsl(c)
def visualize_features(filename, user_features, df_train, features):
features_map = dict()
for ft, flist in features.items():
for f in flist:
features_map[f] = ft
vals = list()
names = list()
mean_values = df_train.mean().to_dict()
max_values = df_train.max().to_dict()
min_values = df_train.min().to_dict()
for f, v in user_features.items():
if f in ['uid', 'crash']:
continue
if np.isnan(v) or np.isinf(
v) or v == -1 or max_values[f] == min_values[f] or np.isinf(
min_values[f]) or np.isinf(max_values[f]):
vals.append(0)
else:
v1 = (v - min_values[f]) / (max_values[f] - min_values[f])
v2 = (mean_values[f] - min_values[f]) / (max_values[f] -
min_values[f])
d = v1 - v2
vals.append(d)
names.append('%s-%s' % (features_map[f], f))
gap = (max(vals) - min(vals)) / 6
bins = np.arange(min(vals), max(vals), gap)
color_scale = list(
cl.to_numeric(cl.scales[str(len(bins) - 1)]['div']['RdYlGn']))
color_scale = [cl2hex(c) for c in color_scale]
colors = pd.cut(vals, bins=bins, labels=color_scale)
colors = [
c if not isinstance(c, float) else color_scale[3] for c in colors
]
fetures_per_plot = 20
fig = plt.figure(figsize=(50, 40))
fontsize = 13
pid = 0
for cid in range(0, 3):
for rid in range(0, 7):
plt.subplot(3, 7, pid + 1)
ifrom = pid * fetures_per_plot
ito = pid * fetures_per_plot + fetures_per_plot
svals = vals[ifrom:ito]
snames = names[ifrom:ito]
scolors = colors[ifrom:ito]
x = np.arange(len(svals))
plt.barh(x, svals, color=scolors)
for i, v, n in zip(x, svals, snames):
if v < 0:
plt.text(0.01, i, n, fontsize=fontsize)
elif v > 0:
plt.text(-0.01,
i,
n,
horizontalalignment='right',
fontsize=fontsize)
elif v == 0:
plt.text(0.01,
i,
n,
horizontalalignment='center',
fontsize=fontsize)
plt.axvline(0, color='k')
plt.axis('off')
plt.xlim(min(vals), max(vals))
pid += 1
st = fig.suptitle('Final Features', fontsize=fontsize * 10)
fig.tight_layout()
st.set_y(0.95)
fig.subplots_adjust(top=0.85)
plt.savefig(filename, format='png', bbox_inches='tight')
plt.close()
browser = webbrowser.get('chrome')
browser.open('file://%s' % filename)
def plotk_static(self, kval, output_dir):
"""
"""
qvalues = self.kvals[kval].qvals
plt.style.use("ggplot")
numinds = self.number_indv
clist = [[i / 255. for i in x] for x in cl.to_numeric(c)]
# Update plot width according to the number of samples
plt.rcParams["figure.figsize"] = (8 * numinds * .03, 2.64)
fig = plt.figure()
axe = fig.add_subplot(111, xlim=(-.5, numinds - .5), ylim=(0, 1))
for i in range(qvalues.shape[1]):
# Get bar color. If K exceeds the 12 colors, generate random color
try:
clr = clist[i]
except IndexError:
clr = np.random.rand(3, 1)
if i == 0:
axe.bar(range(numinds),
qvalues[:, i],
facecolor=clr,
edgecolor="grey",
width=1)
former_q = qvalues[:, i]
else:
axe.bar(range(numinds),
qvalues[:, i],
bottom=former_q,
facecolor=clr,
edgecolor="grey",
width=1)
former_q = former_q + qvalues[:, i]
# Annotate population info
if self.pops:
pop_lines = list(
OrderedDict.fromkeys([x for y in self.pops_xrange
for x in y]))[1:-1]
for pl in pop_lines:
# Add population delimiting lines
plt.axvline(x=pl - 0.5, linewidth=1.5, color="black")
for p, pos in enumerate(self.pops_xpos):
axe.text(pos,
-0.05,
self.pops[p],
rotation=45,
va="top",
ha="right",
fontsize=16,
weight="bold")
else:
for pos in range(self.number_indv):
axe.text(pos,
-0.05,
self.indv[pos],
rotation=45,
va="top",
ha="right",
fontsize=10)
for axis in ["top", "bottom", "left", "right"]:
axe.spines[axis].set_linewidth(2)
axe.spines[axis].set_color("black")
plt.yticks([])
plt.xticks([])
kfile = self.kvals[kval].file_path
filename = splitext(basename(kfile))[0]
filepath = join(output_dir, filename)
plt.savefig("{}.svg".format(filepath), bbox_inches="tight")
def scatter_plot_with_highlighted_clusters(highlighted_cls_names,
syl_cls_names,
syl_ids,
ordination_data,
pdf=None,
fig=None):
"""
Plot a scatter plot with all datapoints, those which are highlighted are displayed in colour and bigger size
The positions of the datapoints are exactly the same as displayed in Koe's ordination view
:param highlighted_cls_names: list of classes to highlight
:param syl_cls_names: array of class names corresponding to each datapoint
:param syl_ids: array of syllable IDs corresponding to each datapoint
:param ordination_data:
:param pdf: None to display inline, or instance pf PdfPages to write to a pdf file
:return:
"""
import matplotlib.pyplot as plt
nCategoricalColours = 11
nClasses = len(highlighted_cls_names) + 1
if nClasses <= nCategoricalColours:
# colours = cl.to_numeric(cl.scales[str(nClasses)]['div']['Spectral'])
colours = cl.to_numeric(cl.scales[str(nClasses)]['div']['Spectral'])
else:
colours = cl.to_numeric(
cl.interp(cl.scales[str(nCategoricalColours)]['div']['Spectral'],
nClasses))
colours = (np.array(colours) / 255.).tolist()
# Display clustering:
if fig is None:
fig = plt.figure(figsize=(18, 18))
ax = fig.gca()
syl_inds_unused = np.ones((len(syl_ids), ))
for cls, colour in zip(highlighted_cls_names, colours[1:]):
syl_inds = np.where(syl_cls_names == cls)
syl_inds_unused[syl_inds] = 0
x = ordination_data[syl_inds, 0]
y = ordination_data[syl_inds, 1]
c = colour
ax.scatter(x=x,
y=y,
s=100,
c=[c],
edgecolors=(0, 0, 0),
linewidths=1,
label=cls,
alpha=0.5)
syl_inds_unused = np.where(syl_inds_unused == 1)
x = ordination_data[syl_inds_unused, 0]
y = ordination_data[syl_inds_unused, 1]
c = colours[0]
ax.scatter(x=x, y=y, s=10, c=[c], linewidths=0, label='other', alpha=0.2)
plt.legend(loc=2)
if pdf:
pdf.savefig(fig)
plt.close()
else:
plt.show()
def test_to_numeric(self):
scales = cl.to_numeric(cl.scales['3']['div']['RdYlBu'])
self.assertEqual(
scales,
[(252.0, 141.0, 89.0), (255.0, 255.0, 191.0), (145.0, 191.0, 219.0)]
)
from fivepseq.viz.bokeh_plots import bokeh_transcript_scatter_plot
from fivepseq.logic.structures.fivepseq_counts import CountManager, FivePSeqCounts
dir_5pseq_human = "/proj/sllstore2017018/lilit/5pseq_human"
transcript_assembly = pd.read_csv(os.path.join(dir_5pseq_human,
"fivepseq_Hela-rep1",
"transcript_assembly.txt"),
sep="\t")
hela_chx_counts = CountManager.read_counts_as_list(
os.path.join(dir_5pseq_human, "fivepseq_HelaCHX-rep1",
"counts_FULL_LENGTH.txt"))
high_transcripts = pd.read_csv(
"/proj/sllstore2017018/lilit/5pseq_human/resources/top1000.transcripts.txt"
)
t_ids = transcript_assembly.iloc[:, 0]
t_ids = [w.replace("transcript:", "") for w in t_ids]
chx_high_ind = [i for i, item in enumerate(t_ids) if item in high_transcripts]
bokeh_transcript_scatter_plot(
"HelaCHX-rep1", {"HelaCHX-rep1": hela_chx_counts},
transcript_assembly,
{"HelaCHX-rep1": cl.to_numeric(cl.scales['9']['qual']['Set3'])[2]},
FivePSeqCounts.TERM,
500,
index_filter=chx_high_ind,
min_count=0)
return
graph minus alone or non-conform nodes
'''
import sys
sys.path.insert(0, '/Users/fabien/Documents/workspace/github/policosm')
import matplotlib.pyplot as plt
import colorlover as cl
from policosm.extractors.roadsGraph import *
import policosm.geoNetworks as pocogeo
from policosm.functions import *
colors = [ (r/255,v/255,b/255) for r,v,b in cl.to_numeric(cl.scales['9']['seq']['Purples'])]
edgeWidth=[1,1,1,1,1,2,2,3,3]
def drawRoads(G, edgeColorAttribute='level', edgeColorScale=9, nodeColorAttribute=None, nodeColorScale=None):
pos={}
for n in G.nodes():
pos[n] = (G.node[n]['longitude'],G.node[n]['latitude'])
# nodes
nx.draw_networkx_nodes(G,pos,node_size=0,alpha=0.8,color=colors[-1])
# edges
for i in range(0,9):
selectedEdges = [(u,v) for (u,v,d) in G.edges(data=True) if d['level'] == i]
selectedColors = [colors[i] for e in selectedEdges]
def create_raster_plot_combined(trials, align_event,
sorting_var='trial_id',
x_lim=[-1, 1],
show_plot=False,
fig_dir=None,
store_type=None):
sorting_query, mark, label = get_sort_and_marker(
align_event, sorting_var)
fig = plt.figure(dpi=150, frameon=False, figsize=[10, 5])
ax = plt.Axes(fig, [0., 0., 1., 1.])
if len(trials):
if sorting_var == 'trial_id':
spk_times, trial_ids = (trials & 'event="{}"'.format(align_event)).fetch(
'trial_spike_times', 'trial_id', order_by='trial_id')
spk_trial_ids = np.hstack(
[[trial_id] * len(spk_time)
for trial_id, spk_time in enumerate(spk_times)])
ax.plot(np.hstack(spk_times), spk_trial_ids, 'k.', alpha=0.5,
markeredgewidth=0)
elif sorting_var == 'contrast':
spk_times, trial_contrasts = (trials & 'event="{}"'.format(align_event)).fetch(
'trial_spike_times', 'trial_signed_contrast',
order_by='trial_signed_contrast, trial_id')
spk_trial_ids = np.hstack(
[[trial_id] * len(spk_time)
for trial_id, spk_time in enumerate(spk_times)])
ax.plot(np.hstack(spk_times), spk_trial_ids, 'k.', alpha=0.5,
markeredgewidth=0)
# plot different contrasts as background
contrasts, u_inds = np.unique(trial_contrasts, return_index=True)
u_inds = list(u_inds) + [len(trial_contrasts)]
tick_positions = np.add(u_inds[1:], u_inds[:-1])/2
puor = cl.scales[str(len(contrasts))]['div']['PuOr']
puor = np.divide(cl.to_numeric(puor), 255)
for i, ind in enumerate(u_inds[:-1]):
ax.fill_between([-1, 1], u_inds[i], u_inds[i+1]-1, color=puor[i], alpha=0.8)
fig.add_axes(ax)
elif sorting_var == 'feedback type':
spk_times, trial_fb_types = (trials & 'event="{}"'.format(align_event)).fetch(
'trial_spike_times', 'trial_feedback_type',
order_by='trial_feedback_type, trial_id')
spk_trial_ids = np.hstack(
[[trial_id] * len(spk_time)
for trial_id, spk_time in enumerate(spk_times)])
ax.plot(np.hstack(spk_times), spk_trial_ids, 'k.', alpha=0.5,
markeredgewidth=0)
# plot different feedback types as background
fb_types, u_inds = np.unique(trial_fb_types, return_index=True)
u_inds = list(u_inds) + [len(trial_fb_types)]
colors = sns.diverging_palette(10, 240, n=len(fb_types))
for i, ind in enumerate(u_inds[:-1]):
ax.fill_between([-1, 1], u_inds[i], u_inds[i+1]-1, color=colors[i], alpha=0.5)
fig.add_axes(ax)
else:
spk_times_left, marking_points_left, \
spk_times_right, marking_points_right, \
spk_times_incorrect, marking_points_incorrect = \
get_spike_times_trials(
trials, sorting_var, align_event, sorting_query, mark)
id_gap = len(trials) * 0
if len(spk_times_incorrect):
spk_times_all_incorrect = np.hstack(spk_times_incorrect)
id_incorrect = [[i] * len(spike_time)
for i, spike_time in
enumerate(spk_times_incorrect)]
id_incorrect = np.hstack(id_incorrect)
ax.plot(spk_times_all_incorrect, id_incorrect, 'r.',
alpha=0.5, markeredgewidth=0, label='incorrect trials')
ax.plot(marking_points_incorrect,
range(len(spk_times_incorrect)), 'r', label=label)
else:
id_incorrect = [0]
if not len(id_incorrect):
id_incorrect = [0]
if len(spk_times_left):
spk_times_all_left = np.hstack(spk_times_left)
id_left = [[i + max(id_incorrect) + id_gap] * len(spike_time)
for i, spike_time in
enumerate(spk_times_left)]
id_left = np.hstack(id_left)
ax.plot(spk_times_all_left, id_left, 'g.',
alpha=0.5, markeredgewidth=0, label='left trials')
ax.plot(marking_points_left,
np.add(range(len(spk_times_left)), max(id_incorrect) + id_gap), 'g')
else:
id_left = [max(id_incorrect)]
if not len(id_left):
id_left = [max(id_incorrect)]
if len(spk_times_right):
spk_times_all_right = np.hstack(spk_times_right)
id_right = [[i + max(id_left) + id_gap] * len(spike_time)
for i, spike_time in enumerate(spk_times_right)]
id_right = np.hstack(id_right)
ax.plot(spk_times_all_right, id_right, 'b.',
alpha=0.5, markeredgewidth=0, label='right trials')
ax.plot(marking_points_right,
np.add(range(len(spk_times_right)), max(id_left) + id_gap), 'b')
else:
id_right = [max(id_left)]
if not len(id_right):
id_right = [max(id_left)]
ax.set_axis_off()
fig.add_axes(ax)
# hide the axis
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# set the limits
ax.set_xlim(x_lim[0], x_lim[1])
if sorting_var in ('trial_id', 'contrast', 'feedback type'):
if len(spk_trial_ids):
y_lim = max(spk_trial_ids) * 1.02
else:
y_lim = 2
else:
y_lim = max(id_right) * 1.02
ax.set_ylim(-2, y_lim)
if not show_plot:
plt.close(fig)
# save the figure with `pad_inches=0` to remove
# any padding in the image
if fig_dir:
store_fig_external(fig, store_type, fig_dir)
fig.clear()
gc.collect()
if sorting_var == 'contrast':
return [0, y_lim], label, contrasts, tick_positions
else:
return [0, y_lim], label
else:
encoded_string = convert_fig_to_encoded_string(fig)
if sorting_var == 'contrast':
return encoded_string, [0, y_lim], label, contrasts, tick_positions
else:
return encoded_string, [0, y_lim], label
class VizPipeline:
FILTER_TOP_POPULATED = "populated"
FILTER_CANONICAL_TRANSCRIPTS = "canonical"
METACOUNTS_TERM = "_metacounts_term"
METACOUNTS_START = "_metacounts_start"
METACOUNTS_TERM_SCALED = "_metacounts_term_scaled"
METACOUNTS_START_SCALED = "_metacounts_start_scaled"
TRIANGLE_TERM = "_triangle_term"
TRIANGLE_START = "_triangle_start"
FRAME_TERM = "_frames_term"
FRAME_START = "_frames_start"
AMINO_ACID_PAUSES = "_amino_acid_pauses"
AMINO_ACID_PAUSES_SCALED = "_amino_acid_pauses_scaled"
FFT_TERM = "_fft_term"
FFT_START = "_fft_start"
count_folders = None
title = "fivepseq_plot_canvas"
png_dir = None
svg_dir = None
supplement_dir = "supplement"
supplement_png_dir = None
supplement_svg_dir = None
logger = logging.getLogger(config.FIVEPSEQ_PLOT_LOGGER)
fivepseq_counts = None
args = None
samples = []
meta_count_term_dict = {}
meta_count_start_dict = {}
count_vector_list_start_dict = {}
count_vector_list_term_dict = {}
amino_acid_df_dict = {}
amino_acid_df_full_dict = {}
codon_df_dict = {}
codon_basesorted_df_dict = {}
frame_count_term_dict = {}
frame_count_start_dict = {}
frame_stats_df_dict = {}
fft_signal_start_dict = {}
fft_signal_term_dict = {}
loci_meta_counts_dict = {}
data_summary_dict = {}
transcript_index = None
combine = False # do not combine plots until data counts are successfully combined
COMBINED = "combined"
meta_count_start_combined = pd.DataFrame()
meta_count_term_combined = pd.DataFrame()
frame_count_START_combined = pd.DataFrame()
frame_count_TERM_combined = pd.DataFrame()
amino_acid_df_combined = pd.DataFrame()
amino_acid_df_full_combined = pd.DataFrame()
codon_df_combined = pd.DataFrame()
codon_basesorted_df_combined = pd.DataFrame()
large_colors_list = (cl.to_numeric(cl.scales['8']['qual']['Paired'][0:6]) +
cl.to_numeric(cl.scales['8']['qual']['Set1'][3:5]))
# cl.to_numeric(cl.scales['5']['qual']['Set3']))
# large_colors_list = ("#771155", "#AA4488", "#114477", "#4477AA", "#117777", "#44AAAA",
# "#777711", "#AAAA44", "#774411", "#AA7744", "#771122", "#AA4455")
colors_dict = None
combined_color_dict = {
COMBINED: cl.to_numeric(cl.scales['9']['qual']['Set3'])[3]
}
phantomjs_installed = None
p_scatter_start = None
p_triangle_term = None
p_scatter_start_scaled = None
p_triangle_term_scaled = None
p_triangle_start = None
p_aa_heatmap = None
p_aa_heatmap_scaled = None
p_frame_barplots_term = None
p_frame_barplots_start = None
p_loci_meta_counts = None
p_fft_plot_start = None
p_fft_plot_term = None
p_scatter_term_combined = None
p_scatter_start_combined = None
p_scatter_term_scaled_combined = None
p_scatter_start_scaled_combined = None
p_triangle_term_combined = None
p_triangle_start_combined = None
p_aa_heatmaps_combined = None
# the distance to be used for plotting amino-acid heatmaps
dist_for_amino_acid_heatmaps = 20
def __init__(self, args, count_folders=None):
"""
Initialize vizualization pipeline with arguments contained in args.
If count_folders are provided explicitely, those will be used instead of sd and md arguments.
:param args:
:param count_folders:
"""
self.args = args
self.count_folders = count_folders
def run(self):
try:
self.prepare_output_folders()
except Exception as e:
err_msg = "Problem with plotting: could not create folders for exporting images: %s" % str(
e)
self.logger.error(err_msg)
raise e
try:
self.process_count_folders()
except Exception as e:
err_msg = "Exception while processing input count folders: %s" % str(
e)
self.logger.error(err_msg)
raise e
try:
self.setup_title()
except Exception as e:
err_msg = "Exception while setting up title for plot canvas: %s" % str(
e)
self.logger.error(err_msg)
raise e
try:
self.initialize_data()
except Exception as e:
err_msg = "Exception while reading data: %s" % str(e)
self.logger.error(err_msg)
raise e
try:
self.plot_multiple_samples()
except Exception as e:
err_msg = "Exception while plotting data: %s" % str(e)
self.logger.error(err_msg)
raise e
try:
self.write_supplement()
except Exception as e:
err_msg = "Exception while writing supplements: %s" % str(e)
self.logger.error(err_msg)
raise e
def process_count_folders(self):
"""
Read input as sd or md (if count_folders is not provided from within fivepseq)
and append the count_folders list in self.
Set up output file title.
:return:
"""
if self.count_folders is None:
if hasattr(self.args, 'sd') and self.args.sd is not None:
if not os.path.isdir(self.args.sd):
err_msg = "Provided sd %s is not a directory" % self.args.sd
self.logger.error(err_msg)
raise Exception(err_msg)
self.count_folders.append(self.args.sd)
elif hasattr(self.args, 'md') and self.args.md is not None:
for d in glob.glob(self.args.md):
if os.path.isdir(d):
if d[-1] == "/":
d = d[0:len(d) - 1]
self.count_folders.append(d)
if len(self.count_folders) == 0:
err_msg = "No count folders provided as input"
self.logger.error(err_msg)
raise Exception(err_msg)
if len(self.count_folders) > 8:
self.logger.info(
"The number of samples exceeds 8 (found %d). Only the first 8 will be plotted"
% len(self.count_folders))
self.count_folders = self.count_folders[0:8]
self.logger.info("The following folders will be used for plotting")
for d in self.count_folders:
if os.path.isdir(d):
self.logger.info("\t%s" % d)
def setup_title(self):
if not hasattr(config.args, 't') or config.args.t is None:
config.args.t = os.path.basename(os.path.dirname(
config.args.b)) + "_" + os.path.basename(config.args.b)
config.args.t.replace("_*", "")
config.args.t.replace("*", "")
self.title = config.args.t
def prepare_output_folders(self):
if not os.path.exists(self.args.o):
try:
os.mkdir(self.args.o)
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.args.o, str(e)))
if self.is_phantomjs_installed():
self.png_dir = os.path.join(self.args.o, "png")
if not os.path.exists(self.png_dir):
try:
os.mkdir(self.png_dir)
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.png_dir, str(e)))
self.svg_dir = os.path.join(self.args.o, "svg")
if not os.path.exists(self.svg_dir):
try:
os.mkdir(self.svg_dir)
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.svg_dir, str(e)))
self.supplement_dir = os.path.join(self.args.o, "supplement")
if not os.path.exists(self.supplement_dir):
try:
os.mkdir(self.supplement_dir)
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.supplement_dir, str(e)))
if self.is_phantomjs_installed():
self.supplement_png_dir = os.path.join(self.supplement_dir, "png")
if not os.path.exists(self.supplement_png_dir):
try:
os.mkdir(os.path.join(self.supplement_dir, "png"))
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.supplement_png_dir, str(e)))
self.supplement_svg_dir = os.path.join(self.supplement_dir, "svg")
if not os.path.exists(os.path.join(self.supplement_dir, "svg")):
try:
os.mkdir(os.path.join(self.supplement_dir, "svg"))
except Exception as e:
raise Exception(
"Output directory %s could not be created: %s" %
(self.supplement_svg_dir, str(e)))
def initialize_data(self):
self.logger.info("Reading data counts.")
for d in self.count_folders:
sample = os.path.basename(d)
self.samples.append(sample)
self.update_dicts(sample, d)
self.colors_dict = dict(
zip(self.samples, self.large_colors_list[0:len(self.samples)]))
self.logger.info("Finished reading data counts.")
if len(self.count_folders) > 1:
try:
self.combine_counts()
self.combine = True
except Exception as e:
err_msg = "Could not combine data counts: %s. Combined plots will not be generated" % str(
e)
self.logger.warn(err_msg)
def update_dicts(self, sample, directory):
self.logger.info("reading counts for sample: %s" % sample)
fivepseq_out = FivePSeqOut(directory)
self.data_summary_dict.update(
{sample: self.read_data_summary(fivepseq_out)})
self.meta_count_start_dict.update(
{sample: self.read_meta_count_start(fivepseq_out)})
self.meta_count_term_dict.update(
{sample: self.read_meta_count_term(fivepseq_out)})
self.frame_count_term_dict.update(
{sample: self.read_frame_count_term(fivepseq_out)})
self.frame_count_start_dict.update(
{sample: self.read_frame_count_start(fivepseq_out)})
self.frame_stats_df_dict.update(
{sample: self.read_frame_stats_df(fivepseq_out)})
self.amino_acid_df_dict.update(
{sample: self.read_amino_acid_df(fivepseq_out, full=False)})
self.amino_acid_df_full_dict.update(
{sample: self.read_amino_acid_df(fivepseq_out, full=True)})
self.codon_df_dict.update(
{sample: self.read_codon_df(fivepseq_out, basesort=False)})
self.codon_basesorted_df_dict.update(
{sample: self.read_codon_df(fivepseq_out, basesort=True)})
self.fft_signal_start_dict.update(
{sample: self.read_fft_signal_start(fivepseq_out)})
self.fft_signal_term_dict.update(
{sample: self.read_fft_signal_term(fivepseq_out)})
self.count_vector_list_start_dict.update(
{sample: self.read_count_vector_list_start(fivepseq_out)})
self.count_vector_list_term_dict.update(
{sample: self.read_count_vector_list_term(fivepseq_out)})
self.loci_meta_counts_dict.update(
{sample: self.read_loci_meta_counts(fivepseq_out)})
if self.args.tf is not None:
filter = self.args.tf
if filter == self.FILTER_TOP_POPULATED:
self.logger.info("Applying filter %s" % filter)
self.transcript_index = CountManager.top_populated_count_vector_indices(
self.count_vector_list_term_dict.get(sample),
self.args.span, 10000)
elif filter == self.FILTER_CANONICAL_TRANSCRIPTS:
self.logger.info("Applying filter %s" % filter)
self.transcript_index = CountManager.canonical_transcript_indices(
directory)
if self.transcript_index is not None:
self.logger.info("Number of filtered transcripts: %d" %
len(self.transcript_index))
self.frame_count_term_dict[sample] = self.frame_count_term_dict[
sample].iloc[self.transcript_index, ]
self.frame_count_start_dict[sample] = self.frame_count_start_dict[
sample].iloc[self.transcript_index, ]
self.frame_stats_df_dict[sample] = None
self.count_vector_list_term_dict[sample] = [
self.count_vector_list_term_dict[sample][i]
for i in self.transcript_index
]
self.count_vector_list_start_dict[sample] = [
self.count_vector_list_start_dict[sample][i]
for i in self.transcript_index
]
self.meta_count_term_dict[
sample] = CountManager.count_vector_to_df(
CountManager.compute_meta_counts(
self.count_vector_list_term_dict[sample]),
FivePSeqCounts.TERM, self.args.span)
self.meta_count_start_dict[
sample] = CountManager.count_vector_to_df(
CountManager.compute_meta_counts(
self.count_vector_list_start_dict[sample]),
FivePSeqCounts.TERM, self.args.span)
# TODO amino acids pauses not subsettable
def combine_counts(self):
self.logger.info("Combining data counts.")
for key in self.meta_count_start_dict.keys():
df_start = self.meta_count_start_dict.get(key)
df_term = self.meta_count_term_dict.get(key)
frame_start = self.frame_count_start_dict.get(key)
frame_term = self.frame_count_term_dict.get(key)
amino_acid_df = self.amino_acid_df_dict.get(key)
amino_acid_df_full = self.amino_acid_df_full_dict.get(key)
codon_df = self.codon_df_dict.get(key)
codon_basesorted_df = self.codon_basesorted_df_dict.get(key)
if len(self.meta_count_start_combined) == 0:
self.meta_count_start_combined = df_start.copy()
self.meta_count_term_combined = df_term.copy()
self.frame_count_START_combined = frame_start.copy()
self.frame_count_TERM_combined = frame_term.copy()
self.amino_acid_df_combined = amino_acid_df.copy()
self.amino_acid_df_full_combined = amino_acid_df_full.copy()
self.codon_df_combined = codon_df.copy()
self.codon_basesorted_df_combined = codon_basesorted_df.copy()
else:
self.meta_count_start_combined.C += df_start.C
self.meta_count_term_combined.C += df_term.C
self.frame_count_START_combined.loc[:, (
'F0', 'F1', 'F2')] += frame_start.loc[:,
('F0', 'F1', 'F2')]
self.frame_count_TERM_combined.loc[:, (
'F0', 'F1', 'F2')] += frame_term.loc[:, ('F0', 'F1', 'F2')]
if amino_acid_df is not None:
self.amino_acid_df_combined += amino_acid_df
if amino_acid_df_full is not None:
self.amino_acid_df_full_combined += amino_acid_df_full
if codon_df is not None:
self.codon_df_combined += codon_df
if codon_basesorted_df is not None:
self.codon_basesorted_df_combined += codon_basesorted_df
def plot_multiple_samples(self):
self.logger.info("Generating plots")
self.make_single_sample_plots(self.title)
figure_list = []
figure_list += [self.p_scatter_start, self.p_scatter_term]
figure_list += [
self.p_scatter_start_scaled, self.p_scatter_term_scaled
]
figure_list += [self.p_triangle_term, self.p_triangle_start]
figure_list += [self.p_aa_heatmap, None]
figure_list += [self.p_aa_heatmap_scaled, None]
figure_list += [self.p_frame_barplots_term, None]
figure_list += [self.p_frame_barplots_start, None]
figure_list += [self.p_fft_plot_start, self.p_fft_plot_term]
if self.p_loci_meta_counts is not None:
figure_list += [self.p_loci_meta_counts, None]
if self.combine:
self.make_combined_plots(self.title)
figure_list += [
self.p_scatter_start_combined, self.p_scatter_term_combined
]
figure_list += [
self.p_scatter_start_scaled_combined,
self.p_scatter_term_scaled_combined
]
figure_list += [
self.p_triangle_term_combined, self.p_triangle_start_combined
]
figure_list += [self.p_aa_heatmaps_combined, None]
bokeh_composite(self.title, figure_list,
os.path.join(self.args.o, self.title + ".html"), 2)
def write_supplement(self):
# codon pauses
self.logger.info("Generating supplement plots: codon pauses")
codon_title = self.title + "_codon_pauses"
if self.combine:
bokeh_composite(
codon_title, [
bokeh_heatmap_grid(
codon_title, self.codon_df_dict, scale=False),
bokeh_heatmap_grid(codon_title + "_scaled",
self.codon_df_dict,
scale=True),
bokeh_heatmap_grid(codon_title + "_combined",
{"combined:": self.codon_df_combined},
scale=False),
bokeh_heatmap_grid(codon_title + "_combined_scaled",
{"combined:": self.codon_df_combined},
scale=True),
bokeh_heatmap_grid(codon_title + "_basesorted",
self.codon_basesorted_df_dict,
scale=False),
bokeh_heatmap_grid(codon_title + "_basesorted_scaled",
self.codon_basesorted_df_dict,
scale=True),
bokeh_heatmap_grid(codon_title + "_basesorted_combined", {
"basesored_combined":
self.codon_basesorted_df_combined
},
scale=False),
bokeh_heatmap_grid(
codon_title + "_basesorted_combined_scaled", {
"basesored_combined":
self.codon_basesorted_df_combined
},
scale=True)
], os.path.join(self.supplement_dir, codon_title + ".html"), 1)
else:
bokeh_composite(
codon_title, [
bokeh_heatmap_grid(
codon_title, self.codon_df_dict, scale=False),
bokeh_heatmap_grid(codon_title + "_scaled",
self.codon_df_dict,
scale=True),
bokeh_heatmap_grid(codon_title + "_basesorted",
self.codon_basesorted_df_dict,
scale=False),
bokeh_heatmap_grid(codon_title + "_basesorted_scaled",
self.codon_basesorted_df_dict,
scale=True)
], os.path.join(self.supplement_dir, codon_title + ".html"), 1)
# amino acid scatter-plots
self.logger.info(
"Generating supplement plots: amino acid scatter-plots")
aa_scatterplots = []
for aa in Codons.AMINO_ACID_TABLE.keys():
self.logger.info("Plotting scatter for %s counts" % aa)
aa_count_dict = {}
for sample in self.samples:
amino_acid_df_full = self.amino_acid_df_full_dict[sample]
aa_df = pd.DataFrame(
data={
'D': map(int, amino_acid_df_full.columns),
'C': amino_acid_df_full.loc[aa, :]
})
aa_df = aa_df.reset_index(drop=True)
aa_count_dict.update({sample: aa_df})
aa_sp = bokeh_scatter_plot(aa,
FivePSeqCounts.TERM,
aa_count_dict,
self.colors_dict,
scale=True,
png_dir=self.supplement_png_dir,
svg_dir=self.supplement_svg_dir)
aa_scatterplots.append(aa_sp)
if self.combine:
aa_df = pd.DataFrame(
data={
'D': map(int,
self.amino_acid_df_full_combined.columns),
'C': self.amino_acid_df_full_combined.loc[aa, :]
})
aa_df = aa_df.reset_index(drop=True)
aa_sp = bokeh_scatter_plot(aa + "_combined",
FivePSeqCounts.TERM,
{"combined": aa_df},
self.combined_color_dict,
scale=True,
png_dir=self.supplement_png_dir,
svg_dir=self.supplement_svg_dir)
aa_scatterplots.append(aa_sp)
bokeh_composite(
self.title + "_amino_acid_scatterplots", aa_scatterplots,
os.path.join(self.supplement_dir,
self.title + "_amino_acid_scatterplots.html"), 2)
def is_phantomjs_installed(self):
if self.phantomjs_installed is not None:
return self.phantomjs_installed
# prepare some data
x = [1, 2, 3, 4, 5]
y = [6, 7, 2, 4, 5]
# output to static HTML file
# output_file("lines.html")
# create a new plot with a title and axis labels
p = figure(title="simple line example",
x_axis_label='x',
y_axis_label='y')
# add a line renderer with legend and line thickness
p.line(x, y, legend="Temp.", line_width=2)
p.output_backend = "svg"
try:
test_file_name = "fivepseq.phantom.test.svg"
export_svgs(p, filename=test_file_name)
self.phantomjs_installed = True
os.remove(test_file_name)
except:
self.phantomjs_installed = False
# TODO in the future fivepseq should attempt to install phantomjs from the very beginning
logging.getLogger(config.FIVEPSEQ_PLOT_LOGGER).warning(
"It seems like phantomjs is not installed no your system. "
"Files may not be exported in svg and png formats, while html will still be available for viewing."
"To install phantomjs, run 'conda install phantomjs selenium pillow'"
)
return self.phantomjs_installed
def read_meta_count_term(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.META_COUNT_TERM_FILE)
try:
meta_count_term = CountManager.read_meta_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
meta_count_term = None
return meta_count_term
def read_meta_count_start(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.META_COUNT_START_FILE)
try:
meta_count_start = CountManager.read_meta_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
meta_count_start = None
return meta_count_start
def read_amino_acid_df(self, fivepseq_out, full=False):
file = fivepseq_out.get_file_path(FivePSeqOut.AMINO_ACID_PAUSES_FILE)
try:
amino_acid_df = CountManager.read_amino_acid_df(file)
if not full:
if amino_acid_df.shape[1] > self.dist_for_amino_acid_heatmaps:
colrange = map(
str,
np.arange(-1 * self.dist_for_amino_acid_heatmaps, 0))
amino_acid_df = amino_acid_df.loc[:, colrange]
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
amino_acid_df = None
return amino_acid_df
def read_codon_df(self, fivepseq_out, basesort=False):
file = fivepseq_out.get_file_path(FivePSeqOut.CODON_PAUSES_FILE)
try:
codon_df = CountManager.read_amino_acid_df(file)
if basesort:
sorted_index = [""] * len(codon_df.index)
for i in range(len(codon_df.index)):
ind = codon_df.index[i]
aa = ind.split("_")[0]
codon = ind.split("_")[1]
sorted_index[i] = codon + "_" + aa
codon_df.index = sorted_index
codon_df = codon_df.reindex(sorted(sorted_index))
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
codon_df = None
return codon_df
def read_frame_count_term(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.FRAME_COUNTS_TERM_FILE)
try:
frame_count_term = CountManager.read_frame_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
frame_count_term = None
return frame_count_term
def read_frame_count_start(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.FRAME_COUNTS_START_FILE)
try:
frame_count_start = CountManager.read_frame_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
frame_count_start = None
return frame_count_start
def read_frame_stats_df(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.FRAME_STATS_DF_FILE)
if os.path.exists(file):
frame_stats_df = pd.read_csv(file, sep="\t", header=0, index_col=0)
else:
frame_stats_df = None
return frame_stats_df
def read_count_vector_list_term(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.COUNT_TERM_FILE)
try:
count_vector_list_term = CountManager.read_counts_as_list(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
count_vector_list_term = None
return count_vector_list_term
def read_count_vector_list_start(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.COUNT_START_FILE)
try:
count_vector_list_start = CountManager.read_counts_as_list(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
count_vector_list_start = None
return count_vector_list_start
def read_loci_meta_counts(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.LOCI_PAUSES_FILE)
loci_meta_counts = None
if os.path.exists(file):
self.logger.info("Loci count file found")
loci_meta_counts = CountManager.read_meta_counts(file)
return loci_meta_counts
def read_fft_signal_start(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.FFT_SIGNALS_START)
try:
fft_signals_start = CountManager.read_meta_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
fft_signals_start = None
return fft_signals_start
def read_fft_signal_term(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.FFT_SIGNALS_TERM)
try:
fft_signals_term = CountManager.read_meta_counts(file)
except:
self.logger.warn(
"The file %s not found, plots for this will be skipped." %
str(file))
fft_signals_term = None
return fft_signals_term
def read_data_summary(self, fivepseq_out):
file = fivepseq_out.get_file_path(FivePSeqOut.DATA_SUMMARY_FILE)
try:
data_summary = pd.read_csv(file,
sep="\t",
header=None,
index_col=0)
except:
self.logger.warn(
"The file %s was not found, table will not be generated" %
str(file))
data_summary = None
return data_summary
def make_single_sample_plots(self, title):
self.p_scatter_term = bokeh_scatter_plot(title + self.METACOUNTS_TERM,
FivePSeqCounts.TERM,
self.meta_count_term_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_scatter_start = bokeh_scatter_plot(title +
self.METACOUNTS_START,
FivePSeqCounts.START,
self.meta_count_start_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_scatter_term_scaled = bokeh_scatter_plot(
title + self.METACOUNTS_TERM_SCALED,
FivePSeqCounts.TERM,
self.meta_count_term_dict,
self.colors_dict,
scale=True,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_scatter_start_scaled = bokeh_scatter_plot(
title + self.METACOUNTS_START_SCALED,
FivePSeqCounts.START,
self.meta_count_start_dict,
self.colors_dict,
scale=True,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_triangle_term = bokeh_triangle_plot(title + self.TRIANGLE_TERM,
self.frame_count_term_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_triangle_start = bokeh_triangle_plot(
title + self.TRIANGLE_START,
self.frame_count_start_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_aa_heatmap = bokeh_heatmap_grid(title + self.AMINO_ACID_PAUSES,
self.amino_acid_df_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_aa_heatmap_scaled = bokeh_heatmap_grid(
title + self.AMINO_ACID_PAUSES_SCALED,
self.amino_acid_df_dict,
scale=True,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_frame_barplots_term = bokeh_frame_barplots(
title + self.FRAME_TERM,
self.frame_count_term_dict,
self.frame_stats_df_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_frame_barplots_start = bokeh_frame_barplots(
title + self.FRAME_START,
self.frame_count_start_dict,
self.frame_stats_df_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_loci_meta_counts = bokeh_scatter_plot(
title + "_loci_meta_counts",
"loci",
self.loci_meta_counts_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
# fft plots
self.p_fft_plot_start = bokeh_fft_plot(title + self.FFT_START,
"start",
self.fft_signal_start_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_fft_plot_term = bokeh_fft_plot(title + self.FFT_TERM,
"term",
self.fft_signal_term_dict,
self.colors_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
def make_combined_plots(self, title):
# Combined plots
self.p_scatter_term_combined = bokeh_scatter_plot(
title + self.METACOUNTS_TERM + "_combined",
FivePSeqCounts.TERM, {"combined": self.meta_count_term_combined},
self.combined_color_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_scatter_start_combined = bokeh_scatter_plot(
title + self.METACOUNTS_START + "_combined",
FivePSeqCounts.START,
{self.COMBINED: self.meta_count_start_combined},
self.combined_color_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
# Combined plots
self.p_scatter_term_scaled_combined = bokeh_scatter_plot(
title + self.METACOUNTS_TERM_SCALED + "_combined",
FivePSeqCounts.TERM, {"combined": self.meta_count_term_combined},
self.combined_color_dict,
scale=True,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_scatter_start_scaled_combined = bokeh_scatter_plot(
title + self.METACOUNTS_START_SCALED + "_combined",
FivePSeqCounts.START,
{self.COMBINED: self.meta_count_start_combined},
self.combined_color_dict,
scale=True,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_triangle_term_combined = bokeh_triangle_plot(
title + self.TRIANGLE_TERM + "_combined",
{self.COMBINED: self.frame_count_TERM_combined},
self.combined_color_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_triangle_start_combined = bokeh_triangle_plot(
title + self.TRIANGLE_START + "_combined",
{self.COMBINED: self.frame_count_START_combined},
self.combined_color_dict,
png_dir=self.png_dir,
svg_dir=self.svg_dir)
self.p_aa_heatmaps_combined = bokeh_heatmap_grid(
title + self.AMINO_ACID_PAUSES + "_combined",
{self.COMBINED: self.amino_acid_df_combined},
png_dir=self.png_dir,
svg_dir=self.svg_dir)
squashed.head(10)
print(f"""The training set now consists of {len(squashed):,} distinct images,
for a total of {squashed["Distinct Defect Types"].sum():,} labeled mask instances.
Furthermore, we have kept a backup of {len(no_defects):,} images witout defects,
in case our model starts producing a lot of false positives and needs more training
on a background class.""")
""" use a consistent color palette per label throughout the notebook """
import colorlover as cl
# see: https://plot.ly/ipython-notebooks/color-scales/
colors = cl.scales['4']['qual']['Set3']
labels = np.array(range(1, 5))
# combining into a dictionary
palette = dict(zip(labels, np.array(cl.to_numeric(colors))))
# we want counts & frequency of the labels
classes = data.groupby(by='ClassId', as_index=False) \
.agg({'ImageId':'count'}) \
.rename(columns={'ImageId':'Count'})
classes['Frequency'] = round(classes.Count / classes.Count.sum() * 100, 2)
classes['Frequency'] = classes['Frequency'].astype(str) + '%'
# plotly for interactive graphs
fig = go.Figure(data=go.Bar(orientation='h',
x=classes.Count,
y=classes.ClassId,
hovertext=classes.Frequency,
text=classes.Count,
textposition='auto',
# Variables
CAMERAS = [
"CAM_FRONT",
"CAM_BACK",
"CAM_FRONT_ZOOMED",
"CAM_FRONT_LEFT",
"CAM_FRONT_RIGHT",
"CAM_BACK_RIGHT",
"CAM_BACK_LEFT",
]
LIDARS = ["LIDAR_TOP", "LIDAR_FRONT_RIGHT", "LIDAR_FRONT_LEFT"]
NAME2COLOR = dict(
zip(
["bus", "car", "other_vehicle", "pedestrian", "truck"],
cl.to_numeric(cl.scales["5"]["div"]["Spectral"]),
))
# Create Lyft object
lv5 = LyftDataset(data_path="./data",
json_path="./data/train_data",
verbose=True)
# Load a single scene
scene = lv5.scene[0]
token_list = get_token_list(scene)
INITIAL_TOKEN = scene["first_sample_token"]
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.CYBORG])
server = app.server
controls = [
return [rgb_to_hex(item) for item in rgb]
algorithms_arranged = ['OPTIMAL',
'RACE',
'CONTROL',
'ONLINE',
'OFFLINE',
'NUCLEAR',
'HBM',
'CALOREE-NP',
'ADAPT-CONTROL',
'ONLINE-ADAPT',
'NUCLEAR-ADAPT',
'HBM-ADAPT']
reds = rgb_to_hex(cl.to_numeric(cl.scales['6']['seq']['Reds']))
reds.reverse()
BuPus = rgb_to_hex(cl.to_numeric(cl.scales['6']['seq']['BuPu']))
greens = rgb_to_hex(cl.to_numeric(cl.scales['3']['seq']['Greens']))
# plot eff
means_deadline = pd.DataFrame([],columns = algorithms_arranged)
for item in deadline:
df = pd.DataFrame(All_data['eff'][item], columns = algorithms)
means_deadline = means_deadline.append(df.mean(),ignore_index=True)
means, errors = means_deadline.mean(), means_deadline.std()
fig, ax = plt.subplots()
means.plot.bar(yerr=errors, ax=ax, color = reds+BuPus+greens)
# plot latency
def plotk_static(self, kval, output_dir, bw=False, use_ind=False):
"""
Generates a structure plot in svg format.
:param kval: (int) Must match the K value from self.kvals
:param output_dir: (string) Path of the plot file
:param bw: (bool) If True, plots will be generated with patterns instead
of colors to distinguish k groups.
:param show_ind: (bool) If True, and if individual labels were provided
with the --ind option, use those labels instead of population labels
"""
numinds = self.number_indv
clist = [[i / 255. for i in x] for x in cl.to_numeric(c)]
# Update plot width according to the number of samples
plt.rcParams["figure.figsize"] = (8 * numinds * .03, 2.64)
fig = plt.figure()
axe = fig.add_subplot(111, xlim=(-.5, numinds - .5), ylim=(0, 1))
# Transforms the qvals matrix when K = 1. If K > 2, use the
# original matrix
if len(self.kvals[kval].qvals.shape) == 1:
# This list comprehension ensures that the shape of the array
# is (i, 1), where i is the number of samples
qvalues = np.array([[x] for x in self.kvals[kval].qvals])
else:
qvalues = self.kvals[kval].qvals
counter = 0
for i in range(kval):
# Determine color/pattern arguments
kwargs = {}
# Use colors todistinguish k groups
if not bw:
# Get bar color. If K exceeds the 12 colors, generate random
# color
try:
clr = clist[counter]
counter += 1
except IndexError:
counter = 0
clr = clist[counter]
kwargs["facecolor"] = clr
kwargs["edgecolor"] = "grey"
else:
grey_rgb = [(float((i + 1)) / (float(qvalues.shape[1]) + 1))
for _ in range(3)]
kwargs["facecolor"] = grey_rgb
kwargs["edgecolor"] = "white"
if i == 0:
axe.bar(range(numinds), qvalues[:, i],
width=1, label="K {}".format(i + 1), **kwargs)
former_q = qvalues[:, i]
else:
axe.bar(range(numinds), qvalues[:, i], bottom=former_q,
width=1, label="K {}".format(i + 1), **kwargs)
former_q = former_q + qvalues[:, i]
# Annotate population info
if self.pops:
pop_lines = list(OrderedDict.fromkeys(
[x for y in self.pops_xrange for x in y]))[1:-1]
for pl in pop_lines:
# Add population delimiting lines
plt.axvline(x=pl - 0.5, linewidth=1.5, color="black")
if not use_ind:
for p, pos in enumerate(self.pops_xpos):
axe.text(pos, -0.05, self.pops[p],
rotation=45, va="top", ha="right", fontsize=16,
weight="bold")
if use_ind or not self.pops:
for pos in range(self.number_indv):
axe.text(pos, -0.05, self.indv[pos],
rotation=45, va="top", ha="right", fontsize=8)
for axis in ["top", "bottom", "left", "right"]:
axe.spines[axis].set_linewidth(2)
axe.spines[axis].set_color("black")
plt.yticks([])
plt.xticks([])
kfile = self.kvals[kval].file_path
filename = splitext(basename(kfile))[0]
filepath = join(output_dir, filename)
plt.savefig("{}.svg".format(filepath), bbox_inches="tight")
# Clear plot object
plt.clf()
plt.close()
def plot_average_prices(results_dir, output_dir):
"""Plot average prices under different schemes"""
# Prices from different models
p_bau = analysis.get_average_prices(results_dir, 'bau_case.pickle', None,
'PRICES', -1)
p_rep = analysis.get_average_prices(results_dir, 'rep_case.pickle',
'stage_2_rep', 'PRICES', -1)
p_tax = analysis.get_average_prices(results_dir, 'rep_case.pickle',
'stage_1_carbon_tax', 'PRICES', -1)
p_price_dev_mppdc = analysis.get_average_prices(
results_dir, 'mppdc_price_change_deviation_case.pickle',
'stage_3_price_targeting', 'lamb', 1)
p_price_dev_heuristic = analysis.get_average_prices(
results_dir, 'heuristic_price_change_deviation_case.pickle',
'stage_3_price_targeting', 'PRICES', -1)
# Create figures
c = cl.to_numeric(
cl.flipper()['qual']['5']['Set1']
) # ['Accent', 'Dark2', 'Paired', 'Pastel1', 'Pastel2', 'Set1', 'Set2', 'Set3'])
fig, ax = plt.subplots()
ax.plot(p_bau.index.tolist(),
p_bau['average_price_real'].tolist(),
color=scale_rgb(c[1]),
alpha=0.7,
linewidth=0.9)
ax.plot(p_tax.index.tolist(),
p_tax['average_price_real'].tolist(),
color=scale_rgb(c[0]),
alpha=0.7,
linewidth=0.9)
ax.plot(p_rep.index.tolist(),
p_rep['average_price_real'].tolist(),
color=scale_rgb(c[2]),
alpha=0.7,
linewidth=0.9)
ax.plot(p_price_dev_mppdc.index.tolist(),
p_price_dev_mppdc['average_price_real'].tolist(),
color=scale_rgb(c[3]),
alpha=0.7,
linewidth=0.9)
ax.plot(p_price_dev_heuristic.index.tolist(),
p_price_dev_heuristic['average_price_real'].tolist(),
color=scale_rgb(c[4]),
alpha=0.6,
linewidth=0.9)
fig.set_size_inches(3, 2.3)
ax.set_ylabel('Average price ($/MWh)', fontsize=9, labelpad=-0.1)
ax.set_xlabel('Year', fontsize=9)
ax.tick_params(labelsize=8)
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(20))
ax.yaxis.set_minor_locator(MultipleLocator(5))
ax.legend(['BAU', 'Tax', 'REP', 'MPPDC', 'Heuristic'],
fontsize=7,
ncol=2,
frameon=False)
fig.subplots_adjust(left=0.16, bottom=0.18, top=0.98, right=0.98)
fig.savefig(os.path.join(output_dir, 'average_prices.png'))
fig.savefig(os.path.join(output_dir, 'average_prices.pdf'))
plt.show()
import sys
sys.path.insert(0, '/Users/fabien/workspace/github/policosm')
import matplotlib.pyplot as plt
from descartes.patch import PolygonPatch
import colorlover as cl
from shapely.geometry import Polygon, shape
import random
from policosm.extractors.roadsGraph import *
from policosm.extractors.buildingsPolygons import *
import policosm.geoNetworks as pocogeo
from policosm.functions import *
colors = [(r / 255, v / 255, b / 255)
for r, v, b in cl.to_numeric(cl.scales['9']['seq']['Purples'])]
edgeWidth = [1, 1, 1, 1, 1, 2, 2, 3, 3]
def drawRoadsBuildingsMatplotlib(polygons,
G,
displayRatio=1.0,
edgeColorAttribute='level',
edgeColorScale=9,
nodeColorAttribute=None,
nodeColorScale=None):
fig = plt.figure()
ax = fig.add_subplot(111)
pos = {}
for n in G.nodes():
import io
from enum import Enum
import colorlover as cl
import matplotlib.pyplot as plt
colors = cl.to_numeric(cl.scales['12']['qual']['Set3'])
int_colors = []
for color in colors:
int_colors.append((int(color[0]), int(color[1]), int(color[2])))
selected_colors = []
for color in int_colors:
selected_colors.append('#%02x%02x%02x' % color)
class TimePeriod:
def get_value(self):
pass
def get_label(self):
pass
class Duration(TimePeriod, Enum):
WEEK = ('7', False)
MONTH = ('30', False)
QUARTER = ('90', False)
SEMESTER = ('180', False)
YEAR = ('365', False)