def plotter(title,
df,
kind = 'line',
x_label = None,
y_label = None,
style = 'ggplot',
figsize = (8, 4),
save = False,
legend_pos = 'best',
reverse_legend = 'guess',
num_to_plot = 7,
tex = 'try',
colours = 'Accent',
cumulative = False,
pie_legend = True,
partial_pie = False,
show_totals = False,
transparent = False,
output_format = 'png',
interactive = False,
black_and_white = False,
show_p_val = False,
indices = False,
**kwargs):
"""Visualise corpus interrogations.
:param title: A title for the plot
:type title: str
:param df: Data to be plotted
:type df: pandas.core.frame.DataFrame
:param x_label: A label for the x axis
:type x_label: str
:param y_label: A label for the y axis
:type y_label: str
:param kind: The kind of chart to make
:type kind: str ('line'/'bar'/'barh'/'pie'/'area')
:param style: Visual theme of plot
:type style: str ('ggplot'/'bmh'/'fivethirtyeight'/'seaborn-talk'/etc)
:param figsize: Size of plot
:type figsize: tuple (int, int)
:param save: If bool, save with *title* as name; if str, use str as name
:type save: bool/str
:param legend_pos: Where to place legend
:type legend_pos: str ('upper right'/'outside right'/etc)
:param reverse_legend: Reverse the order of the legend
:type reverse_legend: bool
:param num_to_plot: How many columns to plot
:type num_to_plot: int/'all'
:param tex: Use TeX to draw plot text
:type tex: bool
:param colours: Colourmap for lines/bars/slices
:type colours: str
:param cumulative: Plot values cumulatively
:type cumulative: bool
:param pie_legend: Show a legend for pie chart
:type pie_legend: bool
:param partial_pie: Allow plotting of pie slices only
:type partial_pie: bool
:param show_totals: Print sums in plot where possible
:type show_totals: str -- 'legend'/'plot'/'both'
:param transparent: Transparent .png background
:type transparent: bool
:param output_format: File format for saved image
:type output_format: str -- 'png'/'pdf'
:param black_and_white: Create black and white line styles
:type black_and_white: bool
:param show_p_val: Attempt to print p values in legend if contained in df
:type show_p_val: bool
:param indices: To use when plotting "distance from root"
:type indices: bool
:param stacked: When making bar chart, stack bars on top of one another
:type stacked: str
:param filled: For area and bar charts, make every column sum to 100
:type filled: str
:param legend: Show a legend
:type legend: bool
:param rot: Rotate x axis ticks by *rot* degrees
:type rot: int
:param subplots: Plot each column separately
:type subplots: bool
:param layout: Grid shape to use when *subplots* is True
:type layout: tuple -- (int, int)
:param interactive: Experimental interactive options
:type interactive: list -- [1, 2, 3]
:returns: matplotlib figure
"""
import corpkit
import os
try:
from IPython.utils.shimmodule import ShimWarning
import warnings
warnings.simplefilter('ignore', ShimWarning)
except:
pass
import matplotlib as mpl
from matplotlib import rc
# prefer seaborn plotting
try:
import seaborn as sns
except:
pass
if interactive:
import matplotlib.pyplot as plt, mpld3
else:
import matplotlib.pyplot as plt
import pandas
from pandas import DataFrame
import numpy
from time import localtime, strftime
from tests import check_pytex, check_spider, check_t_kinter
if interactive:
import mpld3
import collections
from mpld3 import plugins, utils
from plugins import InteractiveLegendPlugin, HighlightLines
# check what environment we're in
tk = check_t_kinter()
running_python_tex = check_pytex()
running_spider = check_spider()
def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
"""remove extreme values from colourmap --- no pure white"""
import matplotlib.colors as colors
import numpy as np
new_cmap = colors.LinearSegmentedColormap.from_list(
'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),
cmap(np.linspace(minval, maxval, n)))
return new_cmap
def get_savename(imagefolder, save = False, title = False, ext = 'png'):
"""Come up with the savename for the image."""
import os
def urlify(s):
"Turn title into filename"
import re
s = s.lower()
s = re.sub(r"[^\w\s-]", '', s)
s = re.sub(r"\s+", '-', s)
s = re.sub(r"-(textbf|emph|textsc|textit)", '-', s)
return s
# name as
if not ext.startswith('.'):
ext = '.' + ext
if type(save) == str:
savename = os.path.join(imagefolder, (urlify(save) + ext))
#this 'else' is redundant now that title is obligatory
else:
if title:
filename = urlify(title) + ext
savename = os.path.join(imagefolder, filename)
# remove duplicated ext
if savename.endswith('%s%s' % (ext, ext)):
savename = savename.replace('%s%s' % (ext, ext), ext, 1)
return savename
def rename_data_with_total(dataframe, was_series = False, using_tex = False, absolutes = True):
"""adds totals (abs, rel, keyness) to entry name strings"""
if was_series:
where_the_words_are = dataframe.index
else:
where_the_words_are = dataframe.columns
the_labs = []
for w in list(where_the_words_are):
if not absolutes:
if was_series:
perc = dataframe.T[w][0]
else:
the_labs.append(w)
continue
if using_tex:
the_labs.append('%s (%.2f\%%)' % (w, perc))
else:
the_labs.append('%s (%.2f %%)' % (w, perc))
else:
if was_series:
score = dataframe.T[w].sum()
else:
score = dataframe[w].sum()
if using_tex:
the_labs.append('%s (n=%d)' % (w, score))
else:
the_labs.append('%s (n=%d)' % (w, score))
if not was_series:
dataframe.columns = the_labs
else:
vals = list(dataframe[list(dataframe.columns)[0]].values)
dataframe = pandas.DataFrame(vals, index = the_labs)
dataframe.columns = ['Total']
return dataframe
def auto_explode(dataframe, input, was_series = False, num_to_plot = 7):
"""give me a list of strings and i'll output explode option"""
output = [0 for s in range(num_to_plot)]
if was_series:
l = list(dataframe.index)
else:
l = list(dataframe.columns)
if type(input) == str or type(input) == int:
input = [input]
if type(input) == list:
for i in input:
if type(i) == str:
index = l.index(i)
else:
index = i
output[index] = 0.1
return output
# check if we're doing subplots
sbplt = False
if 'subplots' in kwargs:
if kwargs['subplots'] is True:
sbplt = True
kwargs['subplots'] = sbplt
if colours is True:
colours = 'Paired'
# todo: get this dynamically instead.
styles = ['dark_background', 'bmh', 'grayscale', 'ggplot', 'fivethirtyeight', 'matplotlib', False, 'mpl-white']
#if style not in styles:
#raise ValueError('Style %s not found. Use %s' % (str(style), ', '.join(styles)))
if style == 'mpl-white':
try:
sns.set_style("whitegrid")
except:
pass
style = 'matplotlib'
if style is not False and style.startswith('seaborn'):
colours = False
# use 'draggable = True' to make a draggable legend
dragmode = kwargs.get('draggable', False)
kwargs.pop('draggable', None)
if kwargs.get('savepath'):
mpl.rcParams['savefig.directory'] = kwargs.get('savepath')
kwargs.pop('savepath', None)
mpl.rcParams['savefig.bbox'] = 'tight'
mpl.rcParams.update({'figure.autolayout': True})
# try to use tex
# TO DO:
# make some font kwargs here
using_tex = False
mpl.rcParams['font.family'] = 'sans-serif'
mpl.rcParams['text.latex.unicode'] = True
if tex == 'try' or tex is True:
try:
rc('text', usetex=True)
rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
using_tex = True
except:
matplotlib.rc('font', family='sans-serif')
matplotlib.rc('font', serif='Helvetica Neue')
matplotlib.rc('text', usetex='false')
rc('text', usetex=False)
else:
rc('text', usetex=False)
if interactive:
using_tex = False
if show_totals is False:
show_totals = 'none'
# find out what kind of plot we're making, and enable
# or disable interactive values if need be
kwargs['kind'] = kind.lower()
if interactive:
if kwargs['kind'].startswith('bar'):
interactive_types = [3]
elif kwargs['kind'] == 'area':
interactive_types = [2, 3]
elif kwargs['kind'] == 'line':
interactive_types = [2, 3]
elif kwargs['kind'] == 'pie':
interactive_types = None
warnings.warn('Interactive plotting not yet available for pie plots.')
else:
interactive_types = [None]
if interactive is False:
interactive_types = [None]
# find out if pie mode, add autopct format
piemode = False
if kind == 'pie':
piemode = True
# always the best spot for pie
#if legend_pos == 'best':
#legend_pos = 'lower left'
if show_totals.endswith('plot') or show_totals.endswith('both'):
kwargs['pctdistance'] = 0.6
if using_tex:
kwargs['autopct'] = r'%1.1f\%%'
else:
kwargs['autopct'] = '%1.1f%%'
# copy data, make series into df
dataframe = df.copy()
was_series = False
if type(dataframe) == pandas.core.series.Series:
was_series = True
if not cumulative:
dataframe = DataFrame(dataframe)
else:
dataframe = DataFrame(dataframe.cumsum())
else:
# don't know if this is much good.
if cumulative:
dataframe = DataFrame(dataframe.cumsum())
if len(list(dataframe.columns)) == 1:
was_series = True
# attempt to convert x axis to ints:
try:
dataframe.index = [int(i) for i in list(dataframe.index)]
except:
pass
# remove totals and tkinter order
if not was_series and not all(x.lower() == 'total' for x in list(dataframe.columns)):
for name, ax in zip(['Total'] * 2 + ['tkintertable-order'] * 2, [0, 1, 0, 1]):
try:
dataframe = dataframe.drop(name, axis = ax, errors = 'ignore')
except:
pass
else:
dataframe = dataframe.drop('tkintertable-order', errors = 'ignore')
dataframe = dataframe.drop('tkintertable-order', axis = 1, errors = 'ignore')
# look at columns to see if all can be ints, in which case, set up figure
# for depnumming
if not was_series:
if indices == 'guess':
def isint(x):
try:
a = float(x)
b = int(a)
except ValueError or OverflowError:
return False
else:
return a == b
if all([isint(x) is True for x in list(dataframe.columns)]):
indices = True
else:
indices = False
# if depnumming, plot all, transpose, and rename axes
if indices is True:
num_to_plot = 'all'
dataframe = dataframe.T
if y_label is None:
y_label = 'Percentage of all matches'
if x_label is None:
x_label = ''
# set backend?
output_formats = ['svgz', 'ps', 'emf', 'rgba', 'raw', 'pdf', 'svg', 'eps', 'png', 'pgf']
if output_format not in output_formats:
raise ValueError('%s output format not recognised. Must be: %s' % (output_format, ', '.join(output_formats)))
# don't know if these are necessary
if 'pdf' in output_format:
plt.switch_backend(output_format)
if 'pgf' in output_format:
plt.switch_backend(output_format)
if num_to_plot == 'all':
if was_series:
if not piemode:
num_to_plot = len(dataframe)
else:
num_to_plot = len(dataframe)
else:
if not piemode:
num_to_plot = len(list(dataframe.columns))
else:
num_to_plot = len(dataframe.index)
# explode pie, or remove if not piemode
if piemode and not sbplt and kwargs.get('explode'):
kwargs['explode'] = auto_explode(dataframe,
kwargs['explode'],
was_series = was_series,
num_to_plot = num_to_plot)
else:
kwargs.pop('explode', None)
legend = kwargs.get('legend', False)
#cut data short
plotting_a_totals_column = False
if was_series:
if list(dataframe.columns)[0] != 'Total':
try:
can_be_ints = [int(x) for x in list(dataframe.index)]
num_to_plot = len(dataframe)
except:
dataframe = dataframe[:num_to_plot]
elif list(dataframe.columns)[0] == 'Total':
plotting_a_totals_column = True
if not 'legend' in kwargs:
legend = False
num_to_plot = len(dataframe)
else:
dataframe = dataframe.T.head(num_to_plot).T
# remove stats fields, put p in entry text, etc.
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
try:
dataframe = dataframe.drop(statfields, axis = 1, errors = 'ignore')
except:
pass
try:
dataframe.ix['p']
there_are_p_vals = True
except:
there_are_p_vals = False
if show_p_val:
if there_are_p_vals:
newnames = []
for col in list(dataframe.columns):
pval = dataframe[col]['p']
def p_string_formatter(val):
if val < 0.001:
if not using_tex:
return 'p < 0.001'
else:
return r'p $<$ 0.001'
else:
return 'p = %s' % format(val, '.3f')
pstr = p_string_formatter(pval)
newname = '%s (%s)' % (col, pstr)
newnames.append(newname)
dataframe.columns = newnames
dataframe.drop(statfields, axis = 0, inplace = True, errors = 'ignore')
else:
warnings.warn('No p-values calculated to show.\n\nUse sort_by and keep_stats in editor() to generate these values.')
else:
if there_are_p_vals:
dataframe.drop(statfields, axis = 0, inplace = True, errors = 'ignore')
# make and set y label
absolutes = True
if type(dataframe) == pandas.core.frame.DataFrame:
try:
if not all([s.is_integer() for s in dataframe.iloc[0,:].values]):
absolutes = False
except:
pass
else:
if not all([s.is_integer() for s in dataframe.values]):
absolutes = False
# use colormap if need be:
if num_to_plot > 0:
if not was_series:
if kind in ['pie', 'line', 'area']:
if colours:
if not plotting_a_totals_column:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
#else:
if colours:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
if piemode:
if num_to_plot > 0:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
else:
if num_to_plot > 0:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
# multicoloured bar charts
if colours:
if kind.startswith('bar'):
if len(list(dataframe.columns)) == 1:
if not black_and_white:
import numpy as np
the_range = np.linspace(0, 1, num_to_plot)
cmap = plt.get_cmap(colours)
kwargs['colors'] = [cmap(n) for n in the_range]
# make a bar width ... ? ...
#kwargs['width'] = (figsize[0] / float(num_to_plot)) / 1.5
# reversing legend option
if reverse_legend is True:
rev_leg = True
elif reverse_legend is False:
rev_leg = False
# show legend or don't, guess whether to reverse based on kind
if kind in ['bar', 'barh', 'area', 'line', 'pie']:
if was_series:
legend = False
if kind == 'pie':
if pie_legend:
legend = True
else:
legend = False
if kind in ['barh', 'area']:
if reverse_legend == 'guess':
rev_leg = True
if not 'rev_leg' in locals():
rev_leg = False
# the default legend placement
if legend_pos is True:
legend_pos = 'best'
# cut dataframe if just_totals
try:
tst = dataframe['Combined total']
dataframe = dataframe.head(num_to_plot)
except:
pass
# rotate automatically
if 'rot' not in kwargs:
if not was_series:
xvals = [str(i) for i in list(dataframe.index)[:num_to_plot]]
#if 'kind' in kwargs:
#if kwargs['kind'] in ['barh', 'area']:
#xvals = [str(i) for i in list(dataframe.columns)[:num_to_plot]]
else:
xvals = [str(i) for i in list(dataframe.columns)[:num_to_plot]]
if len(max(xvals, key=len)) > 6:
if not piemode:
kwargs['rot'] = 45
# no title for subplots because ugly,
if title and not sbplt:
kwargs['title'] = title
# no interactive subplots yet:
if sbplt and interactive:
import warnings
interactive = False
warnings.warn('No interactive subplots yet, sorry.')
return
# not using pandas for labels or legend anymore.
#kwargs['labels'] = None
#kwargs['legend'] = False
if legend:
if num_to_plot > 6:
if not kwargs.get('ncol'):
kwargs['ncol'] = num_to_plot / 7
# kwarg options go in leg_options
leg_options = {'framealpha': .8,
'shadow': kwargs.get('shadow', False),
'ncol': kwargs.pop('ncol', 1)}
# determine legend position based on this dict
if legend_pos:
possible = {'best': 0, 'upper right': 1, 'upper left': 2, 'lower left': 3, 'lower right': 4,
'right': 5, 'center left': 6, 'center right': 7, 'lower center': 8, 'upper center': 9,
'center': 10, 'o r': 2, 'outside right': 2, 'outside upper right': 2,
'outside center right': 'center left', 'outside lower right': 'lower left'}
if type(legend_pos) == int:
the_loc = legend_pos
elif type(legend_pos) == str:
try:
the_loc = possible[legend_pos]
except KeyError:
raise KeyError('legend_pos value must be one of:\n%s\n or an int between 0-10.' %', '.join(list(possible.keys())))
leg_options['loc'] = the_loc
#weirdness needed for outside plot
if legend_pos in ['o r', 'outside right', 'outside upper right']:
leg_options['bbox_to_anchor'] = (1.02, 1)
if legend_pos == 'outside center right':
leg_options['bbox_to_anchor'] = (1.02, 0.5)
if legend_pos == 'outside lower right':
leg_options['loc'] == 'upper right'
leg_options['bbox_to_anchor'] = (0.5, 0.5)
# a bit of distance between legend and plot for outside legends
if type(legend_pos) == str:
if legend_pos.startswith('o'):
leg_options['borderaxespad'] = 1
if not piemode:
if show_totals.endswith('both') or show_totals.endswith('legend'):
dataframe = rename_data_with_total(dataframe,
was_series = was_series,
using_tex = using_tex,
absolutes = absolutes)
else:
if pie_legend:
if show_totals.endswith('both') or show_totals.endswith('legend'):
dataframe = rename_data_with_total(dataframe,
was_series = was_series,
using_tex = using_tex,
absolutes = absolutes)
if piemode:
if partial_pie:
dataframe = dataframe / 100.0
# some pie things
if piemode:
if not sbplt:
kwargs['y'] = list(dataframe.columns)[0]
if pie_legend:
kwargs['legend'] = False
if was_series:
leg_options['labels'] = list(dataframe.index)
else:
leg_options['labels'] = list(dataframe.columns)
else:
if pie_legend:
kwargs['legend'] = False
if was_series:
leg_options['labels'] = list(dataframe.index)
else:
leg_options['labels'] = list(dataframe.index)
def filler(df):
pby = df.T.copy()
for i in list(pby.columns):
tot = pby[i].sum()
pby[i] = pby[i] * 100.0 / tot
return pby.T
areamode = False
if kind == 'area':
areamode = True
if legend is False:
kwargs['legend'] = False
# line highlighting option for interactive!
if interactive:
if 2 in interactive_types:
if kind == 'line':
kwargs['marker'] = ','
if not piemode:
kwargs['alpha'] = 0.1
# convert dates --- works only in my current case!
if plotting_a_totals_column or not was_series:
try:
can_it_be_int = int(list(dataframe.index)[0])
can_be_int = True
except:
can_be_int = False
if can_be_int:
if 1500 < int(list(dataframe.index)[0]):
if 2050 > int(list(dataframe.index)[0]):
n = pandas.PeriodIndex([d for d in list(dataframe.index)], freq='A')
dataframe = dataframe.set_index(n)
if kwargs.get('filled'):
if areamode or kind.startswith('bar'):
dataframe = filler(dataframe)
kwargs.pop('filled', None)
MARKERSIZE = 4
COLORMAP = {
0: {'marker': None, 'dash': (None,None)},
1: {'marker': None, 'dash': [5,5]},
2: {'marker': "o", 'dash': (None,None)},
3: {'marker': None, 'dash': [1,3]},
4: {'marker': "s", 'dash': [5,2,5,2,5,10]},
5: {'marker': None, 'dash': [5,3,1,2,1,10]},
6: {'marker': 'o', 'dash': (None,None)},
7: {'marker': None, 'dash': [5,3,1,3]},
8: {'marker': "1", 'dash': [1,3]},
9: {'marker': "*", 'dash': [5,5]},
10: {'marker': "2", 'dash': [5,2,5,2,5,10]},
11: {'marker': "s", 'dash': (None,None)}
}
HATCHES = {
0: {'color': '#dfdfdf', 'hatch':"/"},
1: {'color': '#6f6f6f', 'hatch':"\\"},
2: {'color': 'b', 'hatch':"|"},
3: {'color': '#dfdfdf', 'hatch':"-"},
4: {'color': '#6f6f6f', 'hatch':"+"},
5: {'color': 'b', 'hatch':"x"}
}
if black_and_white:
if kind == 'line':
kwargs['linewidth'] = 1
cmap = plt.get_cmap('Greys')
new_cmap = truncate_colormap(cmap, 0.25, 0.95)
if kind == 'bar':
# darker if just one entry
if len(dataframe.columns) == 1:
new_cmap = truncate_colormap(cmap, 0.70, 0.90)
kwargs['colormap'] = new_cmap
class dummy_context_mgr():
"""a fake context for plotting without style
perhaps made obsolete by 'classic' style in new mpl"""
def __enter__(self):
return None
def __exit__(self, one, two, three):
return False
with plt.style.context((style)) if style != 'matplotlib' else dummy_context_mgr():
if not sbplt:
# check if negative values, no stacked if so
if areamode:
kwargs['legend'] = False
if dataframe.applymap(lambda x: x < 0.0).any().any():
kwargs['stacked'] = False
rev_leg = False
ax = dataframe.plot(figsize = figsize, **kwargs)
if areamode:
handles, labels = plt.gca().get_legend_handles_labels()
del handles
del labels
else:
plt.gcf().set_tight_layout(False)
if not piemode:
ax = dataframe.plot(figsize = figsize, **kwargs)
else:
ax = dataframe.plot(figsize = figsize, **kwargs)
handles, labels = plt.gca().get_legend_handles_labels()
plt.legend( handles, labels, loc = leg_options['loc'], bbox_to_anchor = (0,-0.1,1,1),
bbox_transform = plt.gcf().transFigure )
# this line allows layouts with missing plots
# i.e. layout = (5, 2) with only nine plots
plt.gcf().set_tight_layout(False)
if 'rot' in kwargs:
if kwargs['rot'] != 0 and kwargs['rot'] != 90:
labels = [item.get_text() for item in ax.get_xticklabels()]
ax.set_xticklabels(labels, rotation = kwargs['rot'], ha='right')
if transparent:
plt.gcf().patch.set_facecolor('white')
plt.gcf().patch.set_alpha(0)
if black_and_white:
if kind == 'line':
# white background
# change everything to black and white with interesting dashes and markers
c = 0
for line in ax.get_lines():
line.set_color('black')
#line.set_width(1)
line.set_dashes(COLORMAP[c]['dash'])
line.set_marker(COLORMAP[c]['marker'])
line.set_markersize(MARKERSIZE)
c += 1
if c == len(list(COLORMAP.keys())):
c = 0
# draw legend with proper placement etc
if legend:
if not piemode and not sbplt:
if 3 not in interactive_types:
handles, labels = plt.gca().get_legend_handles_labels()
# area doubles the handles and labels. this removes half:
if areamode:
handles = handles[-len(handles) / 2:]
labels = labels[-len(labels) / 2:]
if rev_leg:
handles = handles[::-1]
labels = labels[::-1]
lgd = plt.legend(handles, labels, **leg_options)
if interactive:
# 1 = highlight lines
# 2 = line labels
# 3 = legend switches
ax = plt.gca()
# fails for piemode
lines = ax.lines
handles, labels = plt.gca().get_legend_handles_labels()
if 1 in interactive_types:
plugins.connect(plt.gcf(), HighlightLines(lines))
if 3 in interactive_types:
plugins.connect(plt.gcf(), InteractiveLegendPlugin(lines, labels, alpha_unsel=0.0))
for i, l in enumerate(lines):
y_vals = l.get_ydata()
x_vals = l.get_xdata()
x_vals = [str(x) for x in x_vals]
if absolutes:
ls = ['%s (%s: %d)' % (labels[i], x_val, y_val) for x_val, y_val in zip(x_vals, y_vals)]
else:
ls = ['%s (%s: %.2f%%)' % (labels[i], x_val, y_val) for x_val, y_val in zip(x_vals, y_vals)]
if 2 in interactive_types:
#if 'kind' in kwargs and kwargs['kind'] == 'area':
tooltip_line = mpld3.plugins.LineLabelTooltip(lines[i], labels[i])
mpld3.plugins.connect(plt.gcf(), tooltip_line)
#else:
if kind == 'line':
tooltip_point = mpld3.plugins.PointLabelTooltip(l, labels = ls)
mpld3.plugins.connect(plt.gcf(), tooltip_point)
if piemode:
if not sbplt:
plt.axis('equal')
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# add x label
# this could be revised now!
# if time series period, it's year for now
if type(dataframe.index) == pandas.tseries.period.PeriodIndex:
x_label = 'Year'
if x_label is not False:
if type(x_label) == str:
plt.xlabel(x_label)
else:
check_x_axis = list(dataframe.index)[0] # get first entry# get second entry of first entry (year, count)
try:
if type(dataframe.index) == pandas.tseries.period.PeriodIndex:
x_label = 'Year'
check_x_axis = int(check_x_axis)
if 1500 < check_x_axis < 2050:
x_label = 'Year'
else:
x_label = 'Group'
except:
x_label = 'Group'
if not sbplt:
if not piemode:
plt.xlabel(x_label)
def is_number(s):
"""check if str can be can be made into float/int"""
try:
float(s) # for int, long and float
except ValueError:
try:
complex(s) # for complex
except ValueError:
return False
return True
# for now, always turn off sci notation
from matplotlib.ticker import ScalarFormatter
if type(dataframe.index) != pandas.tseries.period.PeriodIndex:
try:
if all(is_number(s) for s in list(dataframe.index)):
plt.gca().xaxis.set_major_formatter(ScalarFormatter())
except:
pass
try:
if all(is_number(s) for s in list(dataframe.columns)):
plt.gca().yaxis.set_major_formatter(ScalarFormatter())
except:
pass
# y labelling
y_l = False
if not absolutes:
y_l = 'Percentage'
else:
y_l = 'Absolute frequency'
def suplabel(axis,label,label_prop=None,
labelpad=5,
ha='center',va='center'):
''' Add super ylabel or xlabel to the figure
Similar to matplotlib.suptitle
axis - string: "x" or "y"
label - string
label_prop - keyword dictionary for Text
labelpad - padding from the axis (default: 5)
ha - horizontal alignment (default: "center")
va - vertical alignment (default: "center")
'''
fig = plt.gcf()
xmin = []
ymin = []
for ax in fig.axes:
xmin.append(ax.get_position().xmin)
ymin.append(ax.get_position().ymin)
xmin,ymin = min(xmin),min(ymin)
dpi = fig.dpi
if axis.lower() == "y":
rotation=90.
x = xmin-float(labelpad)/dpi
y = 0.5
elif axis.lower() == 'x':
rotation = 0.
x = 0.5
y = ymin - float(labelpad)/dpi
else:
raise Exception("Unexpected axis: x or y")
if label_prop is None:
label_prop = dict()
plt.gcf().text(x,y,label,rotation=rotation,
transform=fig.transFigure,
ha=ha,va=va,
**label_prop)
if y_label is not False:
if not sbplt:
if not piemode:
if type(y_label) == str:
plt.ylabel(y_label)
else:
plt.ylabel(y_l)
else:
if type(y_label) == str:
the_y = y_label
else:
the_y = y_l
#suplabel('y', the_y, labelpad = 1.5)
plt.gcf().text(0.04, 0.5, the_y, va='center', rotation='vertical')
#plt.subplots_adjust(left=0.5)
# if not piemode:
# if type(y_label) == str:
# plt.ylabel(y_label)
# else:
# plt.ylabel(y_l)
# hacky: turn legend into subplot titles :)
if sbplt:
# title the big plot
#plt.gca().suptitle(title, fontsize = 16)
#plt.subplots_adjust(top=0.9)
# get all axes
if 'layout' not in kwargs:
axes = [l for index, l in enumerate(ax)]
else:
axes = []
cols = [l for index, l in enumerate(ax)]
for col in cols:
for bit in col:
axes.append(bit)
# set subplot titles
for index, a in enumerate(axes):
try:
titletext = list(dataframe.columns)[index]
except:
pass
a.set_title(titletext)
try:
a.legend_.remove()
except:
pass
# remove axis labels for pie plots
if piemode:
a.axes.get_xaxis().set_visible(False)
a.axes.get_yaxis().set_visible(False)
a.axis('equal')
# show grid
a.grid(b=kwargs.get('grid', False))
kwargs.pop('grid', None)
# add sums to bar graphs and pie graphs
# doubled right now, no matter
if not sbplt:
if kind.startswith('bar'):
width = ax.containers[0][0].get_width()
# show grid
ax.grid(b=kwargs.get('grid', False))
kwargs.pop('grid', None)
if was_series:
the_y_limit = plt.ylim()[1]
if show_totals.endswith('plot') or show_totals.endswith('both'):
# make plot a bit higher if putting these totals on it
plt.ylim([0,the_y_limit * 1.05])
for i, label in enumerate(list(dataframe.index)):
if len(dataframe.ix[label]) == 1:
score = dataframe.ix[label][0]
else:
if absolutes:
score = dataframe.ix[label].sum()
else:
#import warnings
#warnings.warn("It's not possible to determine total percentage from individual percentages.")
continue
if not absolutes:
plt.annotate('%.2f' % score, (i, score), ha = 'center', va = 'bottom')
else:
plt.annotate(score, (i, score), ha = 'center', va = 'bottom')
else:
the_y_limit = plt.ylim()[1]
if show_totals.endswith('plot') or show_totals.endswith('both'):
for i, label in enumerate(list(dataframe.columns)):
if len(dataframe[label]) == 1:
score = dataframe[label][0]
else:
if absolutes:
score = dataframe[label].sum()
else:
#import warnings
#warnings.warn("It's not possible to determine total percentage from individual percentages.")
continue
if not absolutes:
plt.annotate('%.2f' % score, (i, score), ha = 'center', va = 'bottom')
else:
plt.annotate(score, (i, score), ha = 'center', va = 'bottom')
plt.subplots_adjust(left=0.1)
plt.subplots_adjust(bottom=0.18)
if 'layout' not in kwargs:
if not sbplt:
plt.tight_layout()
if save:
import os
if running_python_tex:
imagefolder = '../images'
else:
imagefolder = 'images'
savename = get_savename(imagefolder, save = save, title = title, ext = output_format)
if not os.path.isdir(imagefolder):
os.makedirs(imagefolder)
# save image and get on with our lives
if legend_pos.startswith('o'):
plt.gcf().savefig(savename, dpi=150, bbox_extra_artists=(lgd,), bbox_inches='tight', format = output_format)
else:
plt.gcf().savefig(savename, dpi=150, format = output_format)
time = strftime("%H:%M:%S", localtime())
if os.path.isfile(savename):
print('\n' + time + ": " + savename + " created.")
else:
raise ValueError("Error making %s." % savename)
if dragmode:
plt.legend().draggable()
if sbplt:
plt.subplots_adjust(right=.8)
plt.subplots_adjust(left=.1)
if not interactive and not running_python_tex and not running_spider \
and not tk:
plt.gcf().show()
return
elif running_spider or tk:
return plt
if interactive:
plt.subplots_adjust(right=.8)
plt.subplots_adjust(left=.1)
try:
ax.legend_.remove()
except:
pass
return mpld3.display()
def editor(interrogation,
operation = None,
denominator = False,
sort_by = False,
keep_stats = False,
keep_top = False,
just_totals = False,
threshold = 'medium',
just_entries = False,
skip_entries = False,
merge_entries = False,
newname = 'combine',
multiple_merge = False,
just_subcorpora = False,
skip_subcorpora = False,
span_subcorpora = False,
merge_subcorpora = False,
new_subcorpus_name = False,
replace_names = False,
projection = False,
remove_above_p = False,
p = 0.05,
revert_year = True,
print_info = True,
spelling = False,
selfdrop = True,
calc_all = True,
**kwargs
):
"""Edit results of interrogations, do keywording, sort, etc.
``just/skip_entries`` and ``just/skip_subcorpora`` can take a few different kinds of input:
* str: treated as regular expression to match
* list:
* of integers: indices to match
* of strings: entries/subcorpora to match
``merge_entries`` and ``merge_subcorpora``, however, are best entered as dicts:
``{newname: criteria, newname2: criteria2}```
where criteria is a string, list, etc.
:param interrogation: Results to edit
:type interrogation: pandas.core.frame.DataFrame
:param operation: Kind of maths to do on inputted lists:
'+', '-', '/', '*', '%': self explanatory
'k': log likelihood (keywords)
'a': get distance metric (for use with interrogator 'a' option)
'd': get percent difference (alternative approach to keywording)
:type operation: str
:param denominator: List of results or totals.
If list of results, for each entry in dataframe 1, locate
entry with same name in dataframe 2, and do maths there
if 'self', do all merging/keeping operations, then use
edited interrogation as denominator
:type denominator: pandas.core.series.Series/pandas.core.frame.DataFrame/dict/'self'
:param sort_by: Calculate slope, stderr, r, p values, then sort by:
increase: highest to lowest slope value
decrease: lowest to highest slope value
turbulent: most change in y axis values
static: least change in y axis values
total/most: largest number first
infreq/least: smallest number first
name: alphabetically
:type sort_by: str
:param keep_stats: Keep/drop stats values from dataframe after sorting
:type keep_stats: bool
:param keep_top: After sorting, remove all but the top *keep_top* results
:type keep_top: int
:param just_totals: Sum each column and work with sums
:type just_totals: bool
:param threshold: When using results list as denominator, drop values occurring
fewer than n times. If not keywording, you can use:
``'high'``: denominator total / 2500
``'medium'``: denominator total / 5000
``'low'``: denominator total / 10000
Note: if keywording, there are smaller default thresholds
:type threshold: int/bool
:param just_entries: Keep matching entries
:type just_entries: see above
:param skip_entries: Skip matching entries
:type skip_entries: see above
:param merge_entries: Merge matching entries
:type merge_entries: see above
:param newname: New name for merged entries
:type newname: str/'combine'
:param just_subcorpora: Keep matching subcorpora
:type just_subcorpora: see above
:param skip_subcorpora: Skip matching subcorpora
:type skip_subcorpora: see above
:param span_subcorpora: If subcorpora are numerically named, span all from *int* to *int2*, inclusive
:type span_subcorpora: tuple -- ``(int, int2)``
:param merge_subcorpora: Merge matching subcorpora
:type merge_subcorpora: see above
:param new_subcorpus_name: Name for merged subcorpora
:type new_subcorpus_name: str/``'combine'``
:param replace_names: Edit result names and then merge duplicate names.
:type replace_names: dict -- ``{criteria: replacement_text}``; str -- a regex to delete from names
:param projection: a to multiply results in subcorpus by n
:type projection: tuple -- ``(subcorpus_name, n)``
:param remove_above_p: Delete any result over p
:type remove_above_p: bool
:param p: set the p value
:type p: float
:param revert_year: when doing linear regression on years, turn annual subcorpora into 1, 2 ...
:type revert_year: bool
:param print_info: Print stuff to console showing what's being edited
:type print_info: bool
:param spelling: Convert/normalise spelling:
:type spelling: str -- ``'US'``/``'UK'``
:param selfdrop: When keywording, try to remove target corpus from reference corpus
:type selfdrop: bool
:param calc_all: When keywording, calculate words that appear in either corpus
:type calc_all: bool
:returns: corpkit.interrogation.Interrogation
"""
# grab arguments, in case we get dict input and have to iterate
locs = locals()
import corpkit
import pandas
import signal
import re
import collections
import pandas as pd
import numpy as np
from pandas import DataFrame, Series
from time import localtime, strftime
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
except NameError:
have_ipython = False
try:
from IPython.display import display, clear_output
except ImportError:
pass
return_conc = False
from interrogation import Interrodict, Interrogation, Concordance
if interrogation.__class__ == Interrodict:
locs.pop('interrogation', None)
from collections import OrderedDict
outdict = OrderedDict()
from editor import editor
for i, (k, v) in enumerate(interrogation.items()):
# only print the first time around
if i != 0:
locs['print_info'] = False
# if df2 is also a dict, get the relevant entry
if type(denominator) == dict or denominator.__class__ == Interrodict:
#if sorted(set([i.lower() for i in list(dataframe1.keys())])) == \
# sorted(set([i.lower() for i in list(denominator.keys())])):
# locs['denominator'] = denominator[k]
if kwargs.get('denominator_totals'):
locs['denominator'] = denominator[k].totals
else:
locs['denominator'] = denominator[k].results
outdict[k] = editor(v.results, **locs)
if print_info:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("\n%s: Finished! Output is a dictionary with keys:\n\n '%s'\n" % (thetime, "'\n '".join(sorted(outdict.keys()))))
return Interrodict(outdict)
elif type(interrogation) in [pandas.core.frame.DataFrame, pandas.core.series.Series]:
dataframe1 = interrogation
elif interrogation.__class__ == Interrogation:
#if interrogation.__dict__.get('concordance', None) is not None:
# concordances = interrogation.concordance
branch = kwargs.pop('branch', 'results')
if branch.lower().startswith('r') :
dataframe1 = interrogation.results
elif branch.lower().startswith('t'):
dataframe1 = interrogation.totals
elif branch.lower().startswith('c'):
dataframe1 = interrogation.concordance
return_conc = True
else:
dataframe1 = interrogation.results
elif interrogation.__class__ == Concordance or \
all(x in list(dataframe1.columns) for x in ['l', 'm', 'r']):
return_conc = True
dataframe1 = interrogation
# hope for the best
else:
dataframe1 = interrogation
the_time_started = strftime("%Y-%m-%d %H:%M:%S")
pd.options.mode.chained_assignment = None
pd.set_option('display.float_format', lambda x: '%.2f' % x)
try:
from tests import check_pytex
except ImportError:
from corpkit.tests import check_pytex
if check_pytex():
print_info = False
def combiney(df, df2, operation = '%', threshold = 'medium', prinf = True):
"""mash df and df2 together in appropriate way"""
totals = False
# delete under threshold
if just_totals:
if using_totals:
if not single_totals:
to_drop = list(df2[df2['Combined total'] < threshold].index)
df = df.drop([e for e in to_drop if e in list(df.index)])
if prinf:
to_show = []
[to_show.append(w) for w in to_drop[:5]]
if len(to_drop) > 10:
to_show.append('...')
[to_show.append(w) for w in to_drop[-5:]]
if len(to_drop) > 0:
print('Removing %d entries below threshold:\n %s' % (len(to_drop), '\n '.join(to_show)))
if len(to_drop) > 10:
print('... and %d more ... \n' % (len(to_drop) - len(to_show) + 1))
else:
print('')
else:
denom = df2
else:
denom = list(df2)
if single_totals:
if operation == '%':
totals = df.sum() * 100.0 / float(df.sum().sum())
df = df * 100.0
try:
df = df.div(denom, axis = 0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: cannot combine DataFrame 1 and 2: different shapes' % thetime)
elif operation == '+':
try:
df = df.add(denom, axis = 0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: cannot combine DataFrame 1 and 2: different shapes' % thetime)
elif operation == '-':
try:
df = df.sub(denom, axis = 0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: cannot combine DataFrame 1 and 2: different shapes' % thetime)
elif operation == '*':
totals = df.sum() * float(df.sum().sum())
try:
df = df.mul(denom, axis = 0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: cannot combine DataFrame 1 and 2: different shapes' % thetime)
elif operation == '/':
try:
totals = df.sum() / float(df.sum().sum())
df = df.div(denom, axis = 0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: cannot combine DataFrame 1 and 2: different shapes' % thetime)
elif operation == 'd':
#df.ix['Combined total'] = df.sum()
#to_drop = to_drop = list(df.T[df.T['Combined total'] < threshold].index)
to_drop = [n for n in list(df.columns) if df[n].sum() < threshold]
df = df.drop([e for e in to_drop if e in list(df.columns)], axis = 1)
#df.drop('Combined total')
if prinf:
to_show = []
[to_show.append(w) for w in to_drop[:5]]
if len(to_drop) > 10:
to_show.append('...')
[to_show.append(w) for w in to_drop[-5:]]
if len(to_drop) > 0:
print('Removing %d entries below threshold:\n %s' % (len(to_drop), '\n '.join(to_show)))
if len(to_drop) > 10:
print('... and %d more ... \n' % (len(to_drop) - len(to_show) + 1))
else:
print('')
# get normalised num in target corpus
norm_in_target = df.div(denom, axis = 0)
# get normalised num in reference corpus, with or without selfdrop
tot_in_ref = df.copy()
for c in list(tot_in_ref.index):
if selfdrop:
tot_in_ref.ix[c] = df.sum() - tot_in_ref.ix[c]
else:
tot_in_ref.ix[c] = df.sum()
norm_in_ref = tot_in_ref.div(df.sum().sum())
df = (norm_in_target - norm_in_ref) / norm_in_ref * 100.0
df = df.replace(float(-100.00), np.nan)
elif operation == 'a':
for c in [c for c in list(df.columns) if int(c) > 1]:
df[c] = df[c] * (1.0 / int(c))
df = df.sum(axis = 1) / df2
elif operation.startswith('c'):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
df = pandas.concat([df, df2], axis = 1)
return df, totals
elif not single_totals:
if not operation.startswith('a'):
# generate totals
if operation == '%':
totals = df.sum() * 100.0 / float(df2.sum().sum())
if operation == '*':
totals = df.sum() * float(df2.sum().sum())
if operation == '/':
totals = df.sum() / float(df2.sum().sum())
if operation.startswith('c'):
# add here the info that merging will not work
# with identical colnames
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
d = pd.concat([df.T, df2.T])
# make index nums
d = d.reset_index()
# sum and remove duplicates
d = d.groupby('index').sum()
dx = d.reset_index('index')
dx.index = list(dx['index'])
df = dx.drop('index', axis = 1).T
for index, entry in enumerate(list(df.columns)):
#p.animate(index)
if operation == '%':
try:
df[entry] = df[entry] * 100.0 / df2[entry]
except:
continue
#df.drop(entry, axis = 1, inplace = True)
#df[entry] = maths_done
elif operation == '+':
try:
df[entry] = df[entry] + df2[entry]
except:
continue
elif operation == '-':
try:
df[entry] = df[entry] - df2[entry]
except:
continue
elif operation == '*':
try:
df[entry] = df[entry] * df2[entry]
except:
continue
elif operation == '/':
try:
df[entry] = df[entry] / df2[entry]
except:
continue
else:
for c in [c for c in list(df.columns) if int(c) > 1]:
df[c] = df[c] * (1.0 / int(c))
df = df.sum(axis = 1) / df2.T.sum()
return df, totals
def parse_input(df, the_input):
"""turn whatever has been passed in into list of words that can
be used as pandas indices---maybe a bad way to go about it"""
parsed_input = False
import re
if the_input == 'all':
the_input = r'.*'
if type(the_input) == int:
try:
the_input = str(the_input)
except:
pass
the_input = [the_input]
elif type(the_input) == str or type(the_input) == str:
try:
regex = re.compile(the_input)
parsed_input = [w for w in list(df) if re.search(regex, w)]
return parsed_input
except:
the_input = [the_input]
if type(the_input) == list:
if type(the_input[0]) == int:
parsed_input = [word for index, word in enumerate(list(df)) if index in the_input]
elif type(the_input[0]) == str or type(the_input[0]) == unicode:
try:
parsed_input = [word for word in the_input if word in df.columns]
except AttributeError: # if series
parsed_input = [word for word in the_input if word in df.index]
return parsed_input
def synonymise(df, pos = 'n'):
"""pass a df and a pos and convert df columns to most common synonyms"""
from nltk.corpus import wordnet as wn
#from dictionaries.taxonomies import taxonomies
from collections import Counter
fixed = []
for w in list(df.columns):
try:
syns = []
for syns in wn.synsets(w, pos = pos):
for w in syns:
synonyms.append(w)
top_syn = Counter(syns).most_common(1)[0][0]
fixed.append(top_syn)
except:
fixed.append(w)
df.columns = fixed
return df
def convert_spell(df, convert_to = 'US', print_info = print_info):
"""turn dataframes into us/uk spelling"""
from dictionaries.word_transforms import usa_convert
if print_info:
print('Converting spelling ... \n')
if convert_to == 'UK':
usa_convert = {v: k for k, v in list(usa_convert.items())}
fixed = []
for val in list(df.columns):
try:
fixed.append(usa_convert[val])
except:
fixed.append(val)
df.columns = fixed
return df
def merge_duplicates(df, print_info = print_info):
if print_info:
print('Merging duplicate entries ... \n')
# now we have to merge all duplicates
for dup in df.columns.get_duplicates():
#num_dupes = len(list(df[dup].columns))
temp = df[dup].sum(axis = 1)
#df = df.drop([dup for d in range(num_dupes)], axis = 1)
df = df.drop(dup, axis = 1)
df[dup] = temp
return df
def name_replacer(df, replace_names, print_info = print_info):
"""replace entry names and merge"""
import re
# double or single nest if need be
if type(replace_names) == str:
replace_names = [(replace_names, '')]
if type(replace_names) != dict:
if type(replace_names[0]) == str:
replace_names = [replace_names]
if type(replace_names) == dict:
replace_names = [(v, k) for k, v in list(replace_names.items())]
for to_find, replacement in replace_names:
if print_info:
try:
print('Replacing "%s" with "%s" ...\n' % (to_find, replacement))
except:
print('Deleting "%s" from entry names ...\n' % (to_find))
to_find = re.compile(to_find)
try:
replacement = replacement
except:
replacement = ''
df.columns = [re.sub(to_find, replacement, l) for l in list(df.columns)]
df = merge_duplicates(df, print_info = False)
return df
def just_these_entries(df, parsed_input, prinf = True):
entries = [word for word in list(df) if word not in parsed_input]
if prinf:
print('Keeping %d entries:\n %s' % (len(parsed_input), '\n '.join(parsed_input[:10])))
if len(parsed_input) > 10:
print('... and %d more ... \n' % (len(parsed_input) - 10))
else:
print('')
df = df.drop(entries, axis = 1)
return df
def skip_these_entries(df, parsed_input, prinf = True):
if prinf:
print('Skipping %d entries:\n %s' % (len(parsed_input), '\n '.join(parsed_input[:10])))
if len(parsed_input) > 10:
print('... and %d more ... \n' % (len(parsed_input) - 10))
else:
print('')
df = df.drop(parsed_input, axis = 1)
return df
def newname_getter(df, parsed_input, newname = 'combine', prinf = True, merging_subcorpora = False):
"""makes appropriate name for merged entries"""
if merging_subcorpora:
if newname is False:
newname = 'combine'
if type(newname) == int:
the_newname = list(df.columns)[newname]
elif type(newname) == str:
if newname == 'combine':
if len(parsed_input) <= 3:
the_newname = '/'.join(parsed_input)
elif len(parsed_input) > 3:
the_newname = '/'.join(parsed_input[:3]) + '...'
else:
the_newname = newname
if newname is False:
# revise this code
import operator
sumdict = {}
for item in parsed_input:
summed = sum(list(df[item]))
sumdict[item] = summed
the_newname = max(iter(sumdict.items()), key=operator.itemgetter(1))[0]
if type(the_newname) != str:
the_newname = str(the_newname, errors = 'ignore')
return the_newname
def merge_these_entries(df, parsed_input, the_newname, prinf = True, merging = 'entries'):
# make new entry with sum of parsed input
if len(parsed_input) == 0:
import warnings
warnings.warn('No %s could be automatically merged.\n' % merging)
else:
if prinf:
print('Merging %d %s as "%s":\n %s' % (len(parsed_input), merging, the_newname, '\n '.join(parsed_input[:10])))
if len(parsed_input) > 10:
print('... and %d more ... \n' % (len(parsed_input) - 10))
else:
print('')
# remove old entries
temp = sum([df[i] for i in parsed_input])
if not multiple_merge:
if type(df) == pandas.core.series.Series:
df = df.drop(parsed_input)
else:
df = df.drop(parsed_input, axis = 1)
df[the_newname] = temp
return df
def just_these_subcorpora(df, lst_of_subcorpora, prinf = True):
if type(lst_of_subcorpora[0]) == int:
lst_of_subcorpora = [str(l) for l in lst_of_subcorpora]
good_years = [subcorpus for subcorpus in list(df.index) if subcorpus in lst_of_subcorpora]
if prinf:
print('Keeping %d subcorpora:\n %s' % (len(good_years), '\n '.join(good_years[:10])))
if len(good_years) > 10:
print('... and %d more ... \n' % (len(good_years) - 10))
else:
print('')
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus not in good_years], axis = 0)
return df
def skip_these_subcorpora(df, lst_of_subcorpora, prinf = True):
if type(lst_of_subcorpora) == int:
lst_of_subcorpora = [lst_of_subcorpora]
if type(lst_of_subcorpora[0]) == int:
lst_of_subcorpora = [str(l) for l in lst_of_subcorpora]
bad_years = [subcorpus for subcorpus in list(df.index) if subcorpus in lst_of_subcorpora]
if len(bad_years) == 0:
import warnings
warnings.warn('No subcorpora skipped.\n')
else:
if prinf:
print('Skipping %d subcorpora:\n %s' % (len(bad_years), '\n '.join([str(i) for i in bad_years[:10]])))
if len(bad_years) > 10:
print('... and %d more ... \n' % (len(bad_years) - 10))
else:
print('')
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus in bad_years], axis = 0)
return df
def span_these_subcorpora(df, lst_of_subcorpora, prinf = True):
"""select only a span of numerical suborpora (first, last)"""
non_totals = [subcorpus for subcorpus in list(df.index)]
good_years = [subcorpus for subcorpus in non_totals if int(subcorpus) >= int(lst_of_subcorpora[0]) and int(subcorpus) <= int(lst_of_subcorpora[-1])]
if len(lst_of_subcorpora) == 0:
import warnings
warnings.warn('Span not identified.\n')
else:
if prinf:
print('Keeping subcorpora:\n %d--%d\n' % (int(lst_of_subcorpora[0]), int(lst_of_subcorpora[-1])))
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus not in good_years], axis = 0)
# retotal needed here
return df
def projector(df, list_of_tuples, prinf = True):
"""project abs values"""
if type(list_of_tuples) == list:
tdict = {}
for a, b in list_of_tuples:
tdict[a] = b
list_of_tuples = tdict
for subcorpus, projection_value in list(list_of_tuples.items()):
if type(subcorpus) == int:
subcorpus = str(subcorpus)
df.ix[subcorpus] = df.ix[subcorpus] * projection_value
if prinf:
if type(projection_value) == float:
print('Projection: %s * %s' % (subcorpus, projection_value))
if type(projection_value) == int:
print('Projection: %s * %d' % (subcorpus, projection_value))
if prinf:
print('')
return df
def do_stats(df):
"""do linregress and add to df"""
try:
from scipy.stats import linregress
except ImportError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print('%s: sort type not available in this verion of corpkit.' % thetime)
return False
#from stats.stats import linregress
entries = []
slopes = []
intercepts = []
rs = []
ps = []
stderrs = []
indices = list(df.index)
first_year = list(df.index)[0]
try:
x = [int(y) - int(first_year) for y in indices]
except ValueError:
x = list(range(len(indices)))
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
for entry in list(df.columns):
entries.append(entry)
y = list(df[entry])
slope, intercept, r, p, stderr = linregress(x, y)
slopes.append(slope)
intercepts.append(intercept)
rs.append(r)
ps.append(p)
stderrs.append(stderr)
sl = pd.DataFrame([slopes, intercepts, rs, ps, stderrs],
index = statfields,
columns = list(df.columns))
df = df.append(sl)
# drop infinites and nans
if operation != 'd':
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
return df
def recalc(df, operation = '%'):
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
"""Add totals to the dataframe1"""
#df.drop('Total', axis = 0, inplace = True)
#df.drop('Total', axis = 1, inplace = True)
try:
df['temp-Total'] = df.drop(statfields).sum(axis = 1)
except:
df['temp-Total'] = df.sum(axis = 1)
df = df.T
try:
df['temp-Total'] = df.drop(statfields).sum(axis = 1)
except:
df['temp-Total'] = df.sum(axis = 1)
df = df.T
return df
def resort(df, sort_by = False, keep_stats = False):
"""sort results, potentially using scipy's linregress"""
# translate options and make sure they are parseable
options = ['total', 'name', 'infreq', 'increase', 'turbulent',
'decrease', 'static', 'most', 'least', 'none', 'p']
if sort_by is True:
sort_by = 'total'
if sort_by == 'most':
sort_by = 'total'
if sort_by == 'least':
sort_by = 'infreq'
if sort_by not in options and sort_by:
raise ValueError("sort_by parameter error: '%s' not recognised. Must be True, False, %s" % (sort_by, ', '.join(options)))
if operation.startswith('k'):
if type(df) == pandas.core.series.Series:
if sort_by == 'total':
df = df.order(ascending = False)
elif sort_by == 'infreq':
df = df.order(ascending = True)
elif sort_by == 'name':
df = df.sort_index()
return df
if just_totals:
if sort_by == 'infreq':
df = df.sort_values(by = 'Combined total', ascending = True, axis = 1)
elif sort_by == 'total':
df = df.sort_values(by = 'Combined total', ascending = False, axis = 1)
elif sort_by == 'name':
df = df.sort_index()
return df
# this is really shitty now that i know how to sort, like in the above
if keep_stats:
df = do_stats(df)
if type(df) == bool:
if df is False:
return False
if sort_by == 'total':
if df1_istotals:
df = df.T
df = recalc(df, operation = operation)
tot = df.ix['temp-Total']
df = df[tot.argsort()[::-1]]
df = df.drop('temp-Total', axis = 0)
df = df.drop('temp-Total', axis = 1)
if df1_istotals:
df = df.T
elif sort_by == 'infreq':
if df1_istotals:
df = df.T
df = recalc(df, operation = operation)
tot = df.ix['temp-Total']
df = df[tot.argsort()]
df = df.drop('temp-Total', axis = 0)
df = df.drop('temp-Total', axis = 1)
if df1_istotals:
df = df.T
elif sort_by == 'name':
# currently case sensitive...
df = df.reindex_axis(sorted(df.columns), axis=1)
elif sort_by == 'p':
df = df.T.sort_values(by='p').T
else:
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
if not keep_stats:
df = do_stats(df)
if type(df) == bool:
if df is False:
return False
slopes = df.ix['slope']
if sort_by == 'increase':
df = df[slopes.argsort()[::-1]]
elif sort_by == 'decrease':
df = df[slopes.argsort()]
elif sort_by == 'static':
df = df[slopes.abs().argsort()]
elif sort_by == 'turbulent':
df = df[slopes.abs().argsort()[::-1]]
if remove_above_p:
# the easy way to do it!
df = df.T
df = df[df['p'] <= p]
df = df.T
# remove stats field by default
if not keep_stats:
df = df.drop(statfields, axis = 0)
return df
def set_threshold(big_list, threshold, prinf = True, for_keywords = False):
if type(threshold) == str:
if threshold.startswith('l'):
denominator = 10000
if threshold.startswith('m'):
denominator = 5000
if threshold.startswith('h'):
denominator = 2500
if type(big_list) == pandas.core.frame.DataFrame:
tot = big_list.sum().sum()
if type(big_list) == pandas.core.series.Series:
tot = big_list.sum()
the_threshold = float(tot) / float(denominator)
#if for_keywords:
#the_threshold = the_threshold / 2
else:
the_threshold = threshold
if prinf:
print('Threshold: %d\n' % the_threshold)
return the_threshold
# copy dataframe to be very safe
df = dataframe1.copy()
# make cols into strings
try:
df.columns = [str(c) for c in list(df.columns)]
except:
pass
if operation is None:
operation = 'None'
# do concordance work
if return_conc:
if just_entries:
if type(just_entries) == int:
just_entries = [just_entries]
if type(just_entries) == str:
df = df[df['m'].str.contains(just_entries)]
if type(just_entries) == list:
if all(type(e) == str for e in just_entries):
mp = df['m'].map(lambda x: x in just_entries)
df = df[mp]
else:
df = df.ix[just_entries]
if skip_entries:
if type(skip_entries) == int:
skip_entries = [skip_entries]
if type(skip_entries) == str:
df = df[~df['m'].str.contains(skip_entries)]
if type(skip_entries) == list:
if all(type(e) == str for e in skip_entries):
mp = df['m'].map(lambda x: x not in skip_entries)
df = df[mp]
else:
df = df.drop(skip_entries, axis = 0)
if just_subcorpora:
if type(just_subcorpora) == int:
just_subcorpora = [just_subcorpora]
if type(just_subcorpora) == str:
df = df[df['c'].str.contains(just_subcorpora)]
if type(just_subcorpora) == list:
if all(type(e) == str for e in just_subcorpora):
mp = df['c'].map(lambda x: x in just_subcorpora)
df = df[mp]
else:
df = df.ix[just_subcorpora]
if skip_subcorpora:
if type(skip_subcorpora) == int:
skip_subcorpora = [skip_subcorpora]
if type(skip_subcorpora) == str:
df = df[~df['c'].str.contains(skip_subcorpora)]
if type(skip_subcorpora) == list:
if all(type(e) == str for e in skip_subcorpora):
mp = df['c'].map(lambda x: x not in skip_subcorpora)
df = df[mp]
else:
df = df.drop(skip_subcorpora, axis = 0)
return Concordance(df)
if print_info:
print('\n***Processing results***\n========================\n')
df1_istotals = False
if type(df) == pandas.core.series.Series:
df1_istotals = True
df = pandas.DataFrame(df)
# if just a single result
else:
df = pandas.DataFrame(df)
if operation.startswith('k'):
if sort_by is False:
if not df1_istotals:
sort_by = 'turbulent'
if df1_istotals:
df = df.T
# figure out if there's a second list
# copy and remove totals if there is
single_totals = True
using_totals = False
outputmode = False
if denominator.__class__ == Interrogation:
try:
denominator = denominator.results
except AttributeError:
denominator = denominator.totals
if denominator is not False and type(denominator) != str:
df2 = denominator.copy()
using_totals = True
if type(df2) == pandas.core.frame.DataFrame:
if len(df2.columns) > 1:
single_totals = False
else:
df2 = pandas.Series(df2)
if operation == 'd':
df2 = df2.sum(axis = 1)
single_totals = True
elif type(df2) == pandas.core.series.Series:
single_totals = True
#if operation == 'k':
#raise ValueError('Keywording requires a DataFrame for denominator. Use "self"?')
else:
raise ValueError('Denominator not recognised.')
else:
if operation in ['k', 'd', 'a', '%', '/', '*', '-', '+']:
denominator = 'self'
if denominator == 'self':
outputmode = True
if operation.startswith('a') or operation.startswith('A'):
if list(df.columns)[0] != '0' and list(df.columns)[0] != 0:
df = df.T
if using_totals:
if not single_totals:
df2 = df2.T
if projection:
# projection shouldn't do anything when working with '%', remember.
df = projector(df, projection)
if using_totals:
df2 = projector(df2, projection)
if spelling:
df = convert_spell(df, convert_to = spelling)
df = merge_duplicates(df, print_info = False)
if not single_totals:
df2 = convert_spell(df2, convert_to = spelling, print_info = False)
df2 = merge_duplicates(df2, print_info = False)
if not df1_istotals:
sort_by = 'total'
if replace_names:
df = name_replacer(df, replace_names)
df = merge_duplicates(df)
if not single_totals:
df2 = name_replacer(df2, print_info = False)
df2 = merge_duplicates(df2, print_info = False)
if not sort_by:
sort_by = 'total'
# remove old stats if they're there:
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
try:
df = df.drop(statfields, axis = 0)
except:
pass
if using_totals:
try:
df2 = df2.drop(statfields, axis = 0)
except:
pass
# remove totals and tkinter order
for name, ax in zip(['Total'] * 2 + ['tkintertable-order'] * 2, [0, 1, 0, 1]):
if name == 'Total' and df1_istotals:
continue
try:
df = df.drop(name, axis = ax, errors = 'ignore')
except:
pass
for name, ax in zip(['Total'] * 2 + ['tkintertable-order'] * 2, [0, 1, 0, 1]):
if name == 'Total' and single_totals:
continue
try:
df2 = df2.drop(name, axis = ax, errors = 'ignore')
except:
pass
# merging: make dicts if they aren't already, so we can iterate
if merge_entries:
if type(merge_entries) != list:
if type(merge_entries) == str or type(merge_entries) == str:
merge_entries = {newname: merge_entries}
# for newname, criteria
for name, the_input in sorted(merge_entries.items()):
the_newname = newname_getter(df, parse_input(df, the_input), newname = name, prinf = print_info)
df = merge_these_entries(df, parse_input(df, the_input), the_newname, prinf = print_info)
if not single_totals:
df2 = merge_these_entries(df2, parse_input(df2, the_input), the_newname, prinf = False)
else:
for i in merge_entries:
the_newname = newname_getter(df, parse_input(df, merge_entries), newname = newname, prinf = print_info)
df = merge_these_entries(df, parse_input(df, merge_entries), the_newname, prinf = print_info)
if not single_totals:
df2 = merge_these_entries(df2, parse_input(df2, merge_entries), the_newname, prinf = False)
if merge_subcorpora:
if type(merge_subcorpora) != dict:
if type(merge_subcorpora) == list:
if type(merge_subcorpora[0]) == tuple:
merge_subcorpora = {x: y for x, y in merge_subcorpora}
elif type(merge_subcorpora[0]) == str or type(merge_subcorpora[0]) == str:
merge_subcorpora = {new_subcorpus_name: [x for x in merge_subcorpora]}
elif type(merge_subcorpora[0]) == int:
merge_subcorpora = {new_subcorpus_name: [str(x) for x in merge_subcorpora]}
else:
merge_subcorpora = {new_subcorpus_name: merge_subcorpora}
for name, the_input in sorted(merge_subcorpora.items()):
the_newname = newname_getter(df.T, parse_input(df.T, the_input),
newname = name,
merging_subcorpora = True,
prinf = print_info)
df = merge_these_entries(df.T, parse_input(df.T, the_input), the_newname, merging = 'subcorpora', prinf = print_info).T
if using_totals:
df2 = merge_these_entries(df2.T, parse_input(df2.T, the_input), the_newname, merging = 'subcorpora', prinf = False).T
if just_subcorpora:
df = just_these_subcorpora(df, just_subcorpora, prinf = print_info)
if using_totals:
df2 = just_these_subcorpora(df2, just_subcorpora, prinf = False)
if skip_subcorpora:
df = skip_these_subcorpora(df, skip_subcorpora, prinf = print_info)
if using_totals:
df2 = skip_these_subcorpora(df2, skip_subcorpora, prinf = False)
if span_subcorpora:
df = span_these_subcorpora(df, span_subcorpora, prinf = print_info)
if using_totals:
df2 = span_these_subcorpora(df2, span_subcorpora, prinf = False)
if just_entries:
df = just_these_entries(df, parse_input(df, just_entries), prinf = print_info)
if not single_totals:
df2 = just_these_entries(df2, parse_input(df2, just_entries), prinf = False)
if skip_entries:
df = skip_these_entries(df, parse_input(df, skip_entries), prinf = print_info)
if not single_totals:
df2 = skip_these_entries(df2, parse_input(df2, skip_entries), prinf = False)
# drop infinites and nans
if operation != 'd':
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
# make just_totals as dataframe
just_one_total_number = False
if just_totals:
df = pd.DataFrame(df.sum(), columns = ['Combined total'])
if using_totals:
if not single_totals:
df2 = pd.DataFrame(df2.sum(), columns = ['Combined total'])
else:
just_one_total_number = True
df2 = df2.sum()
tots = df.sum(axis = 1)
if using_totals or outputmode:
if not operation.startswith('k'):
the_threshold = 0
# set a threshold if just_totals
if outputmode is True:
df2 = df.T.sum()
if not just_totals:
df2.name = 'Total'
else:
df2.name = 'Combined total'
using_totals = True
single_totals = True
if just_totals:
if not single_totals:
the_threshold = set_threshold(df2, threshold, prinf = print_info)
if operation == 'd':
the_threshold = set_threshold(df2, threshold, prinf = print_info)
df, tots = combiney(df, df2, operation = operation, threshold = the_threshold, prinf = print_info)
# if doing keywording...
if operation.startswith('k'):
from keys import keywords
# allow saved dicts to be df2, etc
try:
if denominator == 'self':
df2 = df.copy()
except TypeError:
pass
if type(denominator) == str:
if denominator != 'self':
df2 = denominator
else:
the_threshold = False
df = keywords(df, df2,
selfdrop = selfdrop,
threshold = threshold,
printstatus = print_info,
editing = True,
calc_all = calc_all,
**kwargs)
# eh?
df = df.T
# drop infinites and nans
if operation != 'd':
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
# resort data
if sort_by or keep_stats:
df = resort(df, keep_stats = keep_stats, sort_by = sort_by)
if type(df) == bool:
if df is False:
return 'linregress'
if keep_top:
if not just_totals:
df = df[list(df.columns)[:keep_top]]
else:
df = df.head(keep_top)
if just_totals:
# turn just_totals into series:
df = pd.Series(df['Combined total'], name = 'Combined total')
if df1_istotals:
if operation.startswith('k'):
try:
df = pd.Series(df.ix[dataframe1.name])
df.name = '%s: keyness' % df.name
except:
df = df.iloc[0,:]
df.name = 'keyness' % df.name
# generate totals branch if not percentage results:
# fix me
if df1_istotals or operation.startswith('k'):
if not just_totals:
try:
total = pd.Series(df['Total'], name = 'Total')
except:
pass
total = 'none'
#total = df.copy()
else:
total = 'none'
else:
# might be wrong if using division or something...
try:
total = df.T.sum(axis = 1)
except:
total = 'none'
if type(tots) != pandas.core.frame.DataFrame and type(tots) != pandas.core.series.Series:
total = df.sum(axis = 1)
else:
total = tots
if type(df) == pandas.core.frame.DataFrame:
datatype = df.ix[0].dtype
else:
datatype = df.dtype
# TURN INT COL NAMES INTO STR
try:
df.results.columns = [str(d) for d in list(df.results.columns)]
except:
pass
def add_tkt_index(df):
if type(df) != pandas.core.series.Series:
df = df.T
df = df.drop('tkintertable-order', errors = 'ignore', axis = 0)
df = df.drop('tkintertable-order', errors = 'ignore', axis = 1)
df['tkintertable-order'] = pd.Series([index for index, data in enumerate(list(df.index))], index = list(df.index))
df = df.T
return df
# while tkintertable can't sort rows
try:
from tests import check_t_kinter
except ImportError:
from corpkit.tests import check_t_kinter
tk = check_t_kinter()
if tk:
df = add_tkt_index(df)
if kwargs.get('df1_always_df'):
if type(df) == pandas.core.series.Series:
df = pandas.DataFrame(df)
#outputnames = collections.namedtuple('edited_interrogation', ['query', 'results', 'totals'])
#output = outputnames(the_options, df, total)
# delete non-appearing conc lines
if interrogation.__dict__.get('concordance', None) is None:
lns = None
else:
col_crit = interrogation.concordance['m'].map(lambda x: x in list(df.columns))
ind_crit = interrogation.concordance['c'].map(lambda x: x in list(df.index))
lns = interrogation.concordance[col_crit]
lns = lns.loc[ind_crit]
lns = Concordance(lns)
output = Interrogation(results = df, totals = total, query = locs, concordance = lns)
#print '\nResult (sample)\n'
if print_info:
#if merge_entries or merge_subcorpora or span_subcorpora or just_subcorpora or \
#just_entries or skip_entries or skip_subcorpora or printed_th or projection:
print('***Done!***\n========================\n')
#print df.head().T
#print ''
if operation.startswith('k') or just_totals or df1_istotals:
pd.set_option('display.max_rows', 30)
else:
pd.set_option('display.max_rows', 15)
pd.set_option('display.max_columns', 8)
pd.set_option('max_colwidth',70)
pd.set_option('display.width', 800)
pd.set_option('expand_frame_repr', False)
pd.set_option('display.float_format', lambda x: '%.2f' % x)
return output
def editor(
interrogation,
operation=None,
denominator=False,
sort_by=False,
keep_stats=False,
keep_top=False,
just_totals=False,
threshold="medium",
just_entries=False,
skip_entries=False,
merge_entries=False,
newname="combine",
multiple_merge=False,
just_subcorpora=False,
skip_subcorpora=False,
span_subcorpora=False,
merge_subcorpora=False,
new_subcorpus_name=False,
replace_names=False,
projection=False,
remove_above_p=False,
p=0.05,
revert_year=True,
print_info=True,
spelling=False,
selfdrop=True,
calc_all=True,
**kwargs
):
"""Edit results of interrogations, do keywording, sort, etc.
``just/skip_entries`` and ``just/skip_subcorpora`` can take a few different kinds of input:
* str: treated as regular expression to match
* list:
* of integers: indices to match
* of strings: entries/subcorpora to match
``merge_entries`` and ``merge_subcorpora``, however, are best entered as dicts:
``{newname: criteria, newname2: criteria2}```
where criteria is a string, list, etc.
:param interrogation: Results to edit
:type interrogation: pandas.core.frame.DataFrame
:param operation: Kind of maths to do on inputted lists:
'+', '-', '/', '*', '%': self explanatory
'k': log likelihood (keywords)
'a': get distance metric (for use with interrogator 'a' option)
'd': get percent difference (alternative approach to keywording)
:type operation: str
:param denominator: List of results or totals.
If list of results, for each entry in dataframe 1, locate
entry with same name in dataframe 2, and do maths there
if 'self', do all merging/keeping operations, then use
edited interrogation as denominator
:type denominator: pandas.core.series.Series/pandas.core.frame.DataFrame/dict/'self'
:param sort_by: Calculate slope, stderr, r, p values, then sort by:
increase: highest to lowest slope value
decrease: lowest to highest slope value
turbulent: most change in y axis values
static: least change in y axis values
total/most: largest number first
infreq/least: smallest number first
name: alphabetically
:type sort_by: str
:param keep_stats: Keep/drop stats values from dataframe after sorting
:type keep_stats: bool
:param keep_top: After sorting, remove all but the top *keep_top* results
:type keep_top: int
:param just_totals: Sum each column and work with sums
:type just_totals: bool
:param threshold: When using results list as denominator, drop values occurring
fewer than n times. If not keywording, you can use:
``'high'``: denominator total / 2500
``'medium'``: denominator total / 5000
``'low'``: denominator total / 10000
Note: if keywording, there are smaller default thresholds
:type threshold: int/bool
:param just_entries: Keep matching entries
:type just_entries: see above
:param skip_entries: Skip matching entries
:type skip_entries: see above
:param merge_entries: Merge matching entries
:type merge_entries: see above
:param newname: New name for merged entries
:type newname: str/'combine'
:param just_subcorpora: Keep matching subcorpora
:type just_subcorpora: see above
:param skip_subcorpora: Skip matching subcorpora
:type skip_subcorpora: see above
:param span_subcorpora: If subcorpora are numerically named, span all from *int* to *int2*, inclusive
:type span_subcorpora: tuple -- ``(int, int2)``
:param merge_subcorpora: Merge matching subcorpora
:type merge_subcorpora: see above
:param new_subcorpus_name: Name for merged subcorpora
:type new_subcorpus_name: str/``'combine'``
:param replace_names: Edit result names and then merge duplicate names.
:type replace_names: dict -- ``{criteria: replacement_text}``; str -- a regex to delete from names
:param projection: a to multiply results in subcorpus by n
:type projection: tuple -- ``(subcorpus_name, n)``
:param remove_above_p: Delete any result over p
:type remove_above_p: bool
:param p: set the p value
:type p: float
:param revert_year: when doing linear regression on years, turn annual subcorpora into 1, 2 ...
:type revert_year: bool
:param print_info: Print stuff to console showing what's being edited
:type print_info: bool
:param spelling: Convert/normalise spelling:
:type spelling: str -- ``'US'``/``'UK'``
:param selfdrop: When keywording, try to remove target corpus from reference corpus
:type selfdrop: bool
:param calc_all: When keywording, calculate words that appear in either corpus
:type calc_all: bool
:returns: corpkit.interrogation.Interrogation
"""
# grab arguments, in case we get dict input and have to iterate
locs = locals()
import corpkit
import pandas
import signal
import re
import collections
import pandas as pd
import numpy as np
from pandas import DataFrame, Series
from time import localtime, strftime
try:
get_ipython().getoutput()
except TypeError:
have_ipython = True
except NameError:
have_ipython = False
try:
from IPython.display import display, clear_output
except ImportError:
pass
return_conc = False
from interrogation import Interrodict, Interrogation, Concordance
if interrogation.__class__ == Interrodict:
from collections import OrderedDict
outdict = OrderedDict()
from editor import editor
for i, (k, v) in enumerate(interrogation.items()):
# only print the first time around
if i == 0:
pass
# saved_args['print_info'] = True
else:
locs["print_info"] = False
# if df2 is also a dict, get the relevant entry
if type(denominator) == dict:
if sorted(set([i.lower() for i in list(dataframe1.keys())])) == sorted(
set([i.lower() for i in list(denominator.keys())])
):
locs["denominator"] = denominator[k]
if kwargs.get("use_df2_totals"):
saved_args["denominator"] = denominator[k].totals
outdict[k] = editor(v.results, **saved_args)
if print_info:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print(
"\n%s: Finished! Output is a dictionary with keys:\n\n '%s'\n"
% (thetime, "'\n '".join(sorted(outdict.keys())))
)
return Interrodict(outdict)
elif type(interrogation) in [pandas.core.frame.DataFrame, pandas.core.series.Series]:
dataframe1 = interrogation
elif interrogation.__class__ == Interrogation:
# if interrogation.__dict__.get('concordance', None) is not None:
# concordances = interrogation.concordance
branch = kwargs.pop("branch", "results")
if branch.lower().startswith("r"):
dataframe1 = interrogation.results
elif branch.lower().startswith("t"):
dataframe1 = interrogation.totals
elif branch.lower().startswith("c"):
dataframe1 = interrogation.concordance
return_conc = True
else:
dataframe1 = interrogation.results
elif interrogation.__class__ == Concordance or all(x in list(dataframe1.columns) for x in ["l", "m", "r"]):
return_conc = True
dataframe1 = interrogation
# hope for the best
else:
dataframe1 = interrogation
the_time_started = strftime("%Y-%m-%d %H:%M:%S")
pd.options.mode.chained_assignment = None
pd.set_option("display.float_format", lambda x: "%.2f" % x)
from tests import check_pytex
if check_pytex():
print_info = False
def combiney(df, df2, operation="%", threshold="medium", prinf=True):
"""mash df and df2 together in appropriate way"""
totals = False
# delete under threshold
if just_totals:
if using_totals:
if not single_totals:
to_drop = list(df2[df2["Combined total"] < threshold].index)
df = df.drop([e for e in to_drop if e in list(df.index)])
if prinf:
to_show = []
[to_show.append(w) for w in to_drop[:5]]
if len(to_drop) > 10:
to_show.append("...")
[to_show.append(w) for w in to_drop[-5:]]
if len(to_drop) > 0:
print(
"Removing %d entries below threshold:\n %s" % (len(to_drop), "\n ".join(to_show))
)
if len(to_drop) > 10:
print("... and %d more ... \n" % (len(to_drop) - len(to_show) + 1))
else:
print("")
else:
denom = df2
else:
denom = list(df2)
if single_totals:
if operation == "%":
totals = df.sum() * 100.0 / float(df.sum().sum())
df = df * 100.0
try:
df = df.div(denom, axis=0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: cannot combine DataFrame 1 and 2: different shapes" % thetime)
elif operation == "+":
try:
df = df.add(denom, axis=0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: cannot combine DataFrame 1 and 2: different shapes" % thetime)
elif operation == "-":
try:
df = df.sub(denom, axis=0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: cannot combine DataFrame 1 and 2: different shapes" % thetime)
elif operation == "*":
totals = df.sum() * float(df.sum().sum())
try:
df = df.mul(denom, axis=0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: cannot combine DataFrame 1 and 2: different shapes" % thetime)
elif operation == "/":
try:
totals = df.sum() / float(df.sum().sum())
df = df.div(denom, axis=0)
except ValueError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: cannot combine DataFrame 1 and 2: different shapes" % thetime)
elif operation == "d":
# df.ix['Combined total'] = df.sum()
# to_drop = to_drop = list(df.T[df.T['Combined total'] < threshold].index)
to_drop = [n for n in list(df.columns) if df[n].sum() < threshold]
df = df.drop([e for e in to_drop if e in list(df.columns)], axis=1)
# df.drop('Combined total')
if prinf:
to_show = []
[to_show.append(w) for w in to_drop[:5]]
if len(to_drop) > 10:
to_show.append("...")
[to_show.append(w) for w in to_drop[-5:]]
if len(to_drop) > 0:
print("Removing %d entries below threshold:\n %s" % (len(to_drop), "\n ".join(to_show)))
if len(to_drop) > 10:
print("... and %d more ... \n" % (len(to_drop) - len(to_show) + 1))
else:
print("")
# get normalised num in target corpus
norm_in_target = df.div(denom, axis=0)
# get normalised num in reference corpus, with or without selfdrop
tot_in_ref = df.copy()
for c in list(tot_in_ref.index):
if selfdrop:
tot_in_ref.ix[c] = df.sum() - tot_in_ref.ix[c]
else:
tot_in_ref.ix[c] = df.sum()
norm_in_ref = tot_in_ref.div(df.sum().sum())
df = (norm_in_target - norm_in_ref) / norm_in_ref * 100.0
df = df.replace(float(-100.00), np.nan)
elif operation == "a":
for c in [c for c in list(df.columns) if int(c) > 1]:
df[c] = df[c] * (1.0 / int(c))
df = df.sum(axis=1) / df2
elif operation.startswith("c"):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
df = pandas.concat([df, df2], axis=1)
return df, totals
elif not single_totals:
if not operation.startswith("a"):
# generate totals
if operation == "%":
totals = df.sum() * 100.0 / float(df2.sum().sum())
if operation == "*":
totals = df.sum() * float(df2.sum().sum())
if operation == "/":
totals = df.sum() / float(df2.sum().sum())
if operation.startswith("c"):
# add here the info that merging will not work
# with identical colnames
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
d = pd.concat([df.T, df2.T])
# make index nums
d = d.reset_index()
# sum and remove duplicates
d = d.groupby("index").sum()
dx = d.reset_index("index")
dx.index = list(dx["index"])
df = dx.drop("index", axis=1).T
for index, entry in enumerate(list(df.columns)):
# p.animate(index)
if operation == "%":
try:
df[entry] = df[entry] * 100.0 / df2[entry]
except:
continue
# df.drop(entry, axis = 1, inplace = True)
# df[entry] = maths_done
elif operation == "+":
try:
df[entry] = df[entry] + df2[entry]
except:
continue
elif operation == "-":
try:
df[entry] = df[entry] - df2[entry]
except:
continue
elif operation == "*":
try:
df[entry] = df[entry] * df2[entry]
except:
continue
elif operation == "/":
try:
df[entry] = df[entry] / df2[entry]
except:
continue
else:
for c in [c for c in list(df.columns) if int(c) > 1]:
df[c] = df[c] * (1.0 / int(c))
df = df.sum(axis=1) / df2.T.sum()
return df, totals
def parse_input(df, the_input):
"""turn whatever has been passed in into list of words that can
be used as pandas indices---maybe a bad way to go about it"""
# FIX MERGE ERROR HERE
parsed_input = False
import re
if the_input == "all":
the_input = r".*"
if type(the_input) == int:
try:
the_input = str(the_input)
except:
pass
the_input = [the_input]
elif type(the_input) == str or type(the_input) == str:
try:
regex = re.compile(the_input)
parsed_input = [w for w in list(df) if re.search(regex, w)]
return parsed_input
except:
the_input = [the_input]
if type(the_input) == list:
if type(the_input[0]) == int:
parsed_input = [word for index, word in enumerate(list(df)) if index in the_input]
elif type(the_input[0]) == str or type(the_input[0]) == str:
try:
parsed_input = [word for word in the_input if word in df.columns]
except AttributeError: # if series
parsed_input = [word for word in the_input if word in df.index]
return parsed_input
def synonymise(df, pos="n"):
"""pass a df and a pos and convert df columns to most common synonyms"""
from nltk.corpus import wordnet as wn
# from dictionaries.taxonomies import taxonomies
from collections import Counter
fixed = []
for w in list(df.columns):
try:
syns = []
for syns in wn.synsets(w, pos=pos):
for w in syns:
synonyms.append(w)
top_syn = Counter(syns).most_common(1)[0][0]
fixed.append(top_syn)
except:
fixed.append(w)
df.columns = fixed
return df
def convert_spell(df, convert_to="US", print_info=print_info):
"""turn dataframes into us/uk spelling"""
from dictionaries.word_transforms import usa_convert
if print_info:
print("Converting spelling ... \n")
if convert_to == "UK":
usa_convert = {v: k for k, v in list(usa_convert.items())}
fixed = []
for val in list(df.columns):
try:
fixed.append(usa_convert[val])
except:
fixed.append(val)
df.columns = fixed
return df
def merge_duplicates(df, print_info=print_info):
if print_info:
print("Merging duplicate entries ... \n")
# now we have to merge all duplicates
for dup in df.columns.get_duplicates():
# num_dupes = len(list(df[dup].columns))
temp = df[dup].sum(axis=1)
# df = df.drop([dup for d in range(num_dupes)], axis = 1)
df = df.drop(dup, axis=1)
df[dup] = temp
return df
def name_replacer(df, replace_names, print_info=print_info):
"""replace entry names and merge"""
import re
# double or single nest if need be
if type(replace_names) == str:
replace_names = [(replace_names, "")]
if type(replace_names) != dict:
if type(replace_names[0]) == str:
replace_names = [replace_names]
if type(replace_names) == dict:
replace_names = [(v, k) for k, v in list(replace_names.items())]
for to_find, replacement in replace_names:
if print_info:
try:
print('Replacing "%s" with "%s" ...\n' % (to_find, replacement))
except:
print('Deleting "%s" from entry names ...\n' % (to_find))
to_find = re.compile(to_find)
try:
replacement = replacement
except:
replacement = ""
df.columns = [re.sub(to_find, replacement, l) for l in list(df.columns)]
df = merge_duplicates(df, print_info=False)
return df
def just_these_entries(df, parsed_input, prinf=True):
entries = [word for word in list(df) if word not in parsed_input]
if prinf:
print("Keeping %d entries:\n %s" % (len(parsed_input), "\n ".join(parsed_input[:10])))
if len(parsed_input) > 10:
print("... and %d more ... \n" % (len(parsed_input) - 10))
else:
print("")
df = df.drop(entries, axis=1)
return df
def skip_these_entries(df, parsed_input, prinf=True):
if prinf:
print("Skipping %d entries:\n %s" % (len(parsed_input), "\n ".join(parsed_input[:10])))
if len(parsed_input) > 10:
print("... and %d more ... \n" % (len(parsed_input) - 10))
else:
print("")
df = df.drop(parsed_input, axis=1)
return df
def newname_getter(df, parsed_input, newname="combine", prinf=True, merging_subcorpora=False):
"""makes appropriate name for merged entries"""
if merging_subcorpora:
if newname is False:
newname = "combine"
if type(newname) == int:
the_newname = list(df.columns)[newname]
elif type(newname) == str:
if newname == "combine":
if len(parsed_input) <= 3:
the_newname = "/".join(parsed_input)
elif len(parsed_input) > 3:
the_newname = "/".join(parsed_input[:3]) + "..."
else:
the_newname = newname
if newname is False:
# revise this code
import operator
sumdict = {}
for item in parsed_input:
summed = sum(list(df[item]))
sumdict[item] = summed
the_newname = max(iter(sumdict.items()), key=operator.itemgetter(1))[0]
if type(the_newname) != str:
the_newname = str(the_newname, errors="ignore")
return the_newname
def merge_these_entries(df, parsed_input, the_newname, prinf=True, merging="entries"):
# make new entry with sum of parsed input
if len(parsed_input) == 0:
import warnings
warnings.warn("No %s could be automatically merged.\n" % merging)
else:
if prinf:
print(
'Merging %d %s as "%s":\n %s'
% (len(parsed_input), merging, the_newname, "\n ".join(parsed_input[:10]))
)
if len(parsed_input) > 10:
print("... and %d more ... \n" % (len(parsed_input) - 10))
else:
print("")
# remove old entries
temp = sum([df[i] for i in parsed_input])
if not multiple_merge:
if type(df) == pandas.core.series.Series:
df = df.drop(parsed_input)
else:
df = df.drop(parsed_input, axis=1)
df[the_newname] = temp
return df
def just_these_subcorpora(df, lst_of_subcorpora, prinf=True):
if type(lst_of_subcorpora[0]) == int:
lst_of_subcorpora = [str(l) for l in lst_of_subcorpora]
good_years = [subcorpus for subcorpus in list(df.index) if subcorpus in lst_of_subcorpora]
if prinf:
print("Keeping %d subcorpora:\n %s" % (len(good_years), "\n ".join(good_years[:10])))
if len(good_years) > 10:
print("... and %d more ... \n" % (len(good_years) - 10))
else:
print("")
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus not in good_years], axis=0)
return df
def skip_these_subcorpora(df, lst_of_subcorpora, prinf=True):
if type(lst_of_subcorpora) == int:
lst_of_subcorpora = [lst_of_subcorpora]
if type(lst_of_subcorpora[0]) == int:
lst_of_subcorpora = [str(l) for l in lst_of_subcorpora]
bad_years = [subcorpus for subcorpus in list(df.index) if subcorpus in lst_of_subcorpora]
if len(bad_years) == 0:
import warnings
warnings.warn("No subcorpora skipped.\n")
else:
if prinf:
print(
"Skipping %d subcorpora:\n %s"
% (len(bad_years), "\n ".join([str(i) for i in bad_years[:10]]))
)
if len(bad_years) > 10:
print("... and %d more ... \n" % (len(bad_years) - 10))
else:
print("")
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus in bad_years], axis=0)
return df
def span_these_subcorpora(df, lst_of_subcorpora, prinf=True):
"""select only a span of numerical suborpora (first, last)"""
non_totals = [subcorpus for subcorpus in list(df.index)]
good_years = [
subcorpus
for subcorpus in non_totals
if int(subcorpus) >= int(lst_of_subcorpora[0]) and int(subcorpus) <= int(lst_of_subcorpora[-1])
]
if len(lst_of_subcorpora) == 0:
import warnings
warnings.warn("Span not identified.\n")
else:
if prinf:
print("Keeping subcorpora:\n %d--%d\n" % (int(lst_of_subcorpora[0]), int(lst_of_subcorpora[-1])))
df = df.drop([subcorpus for subcorpus in list(df.index) if subcorpus not in good_years], axis=0)
# retotal needed here
return df
def projector(df, list_of_tuples, prinf=True):
"""project abs values"""
if type(list_of_tuples) == list:
tdict = {}
for a, b in list_of_tuples:
tdict[a] = b
list_of_tuples = tdict
for subcorpus, projection_value in list(list_of_tuples.items()):
if type(subcorpus) == int:
subcorpus = str(subcorpus)
df.ix[subcorpus] = df.ix[subcorpus] * projection_value
if prinf:
if type(projection_value) == float:
print("Projection: %s * %s" % (subcorpus, projection_value))
if type(projection_value) == int:
print("Projection: %s * %d" % (subcorpus, projection_value))
if prinf:
print("")
return df
def do_stats(df):
"""do linregress and add to df"""
try:
from scipy.stats import linregress
except ImportError:
from time import localtime, strftime
thetime = strftime("%H:%M:%S", localtime())
print("%s: sort type not available in this verion of corpkit." % thetime)
return False
# from stats.stats import linregress
entries = []
slopes = []
intercepts = []
rs = []
ps = []
stderrs = []
indices = list(df.index)
first_year = list(df.index)[0]
try:
x = [int(y) - int(first_year) for y in indices]
except ValueError:
x = list(range(len(indices)))
statfields = ["slope", "intercept", "r", "p", "stderr"]
for entry in list(df.columns):
entries.append(entry)
y = list(df[entry])
slope, intercept, r, p, stderr = linregress(x, y)
slopes.append(slope)
intercepts.append(intercept)
rs.append(r)
ps.append(p)
stderrs.append(stderr)
sl = pd.DataFrame([slopes, intercepts, rs, ps, stderrs], index=statfields, columns=list(df.columns))
df = df.append(sl)
# drop infinites and nans
if operation != "d":
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
return df
def recalc(df, operation="%"):
statfields = ["slope", "intercept", "r", "p", "stderr"]
"""Add totals to the dataframe1"""
# df.drop('Total', axis = 0, inplace = True)
# df.drop('Total', axis = 1, inplace = True)
try:
df["temp-Total"] = df.drop(statfields).sum(axis=1)
except:
df["temp-Total"] = df.sum(axis=1)
df = df.T
try:
df["temp-Total"] = df.drop(statfields).sum(axis=1)
except:
df["temp-Total"] = df.sum(axis=1)
df = df.T
return df
def resort(df, sort_by=False, keep_stats=False):
"""sort results, potentially using scipy's linregress"""
# translate options and make sure they are parseable
options = [
"total",
"name",
"infreq",
"increase",
"turbulent",
"decrease",
"static",
"most",
"least",
"none",
"p",
]
if sort_by is True:
sort_by = "total"
if sort_by == "most":
sort_by = "total"
if sort_by == "least":
sort_by = "infreq"
if sort_by not in options and sort_by:
raise ValueError(
"sort_by parameter error: '%s' not recognised. Must be True, False, %s" % (sort_by, ", ".join(options))
)
if operation.startswith("k"):
if type(df) == pandas.core.series.Series:
if sort_by == "total":
df = df.order(ascending=False)
elif sort_by == "infreq":
df = df.order(ascending=True)
elif sort_by == "name":
df = df.sort_index()
return df
if just_totals:
if sort_by == "infreq":
df = df.sort_values(by="Combined total", ascending=True, axis=1)
elif sort_by == "total":
df = df.sort_values(by="Combined total", ascending=False, axis=1)
elif sort_by == "name":
df = df.sort_index()
return df
# this is really shitty now that i know how to sort, like in the above
if keep_stats:
df = do_stats(df)
if type(df) == bool:
if df is False:
return False
if sort_by == "total":
if df1_istotals:
df = df.T
df = recalc(df, operation=operation)
tot = df.ix["temp-Total"]
df = df[tot.argsort()[::-1]]
df = df.drop("temp-Total", axis=0)
df = df.drop("temp-Total", axis=1)
if df1_istotals:
df = df.T
elif sort_by == "infreq":
if df1_istotals:
df = df.T
df = recalc(df, operation=operation)
tot = df.ix["temp-Total"]
df = df[tot.argsort()]
df = df.drop("temp-Total", axis=0)
df = df.drop("temp-Total", axis=1)
if df1_istotals:
df = df.T
elif sort_by == "name":
# currently case sensitive...
df = df.reindex_axis(sorted(df.columns), axis=1)
elif sort_by == "p":
df = df.T.sort_values(by="p").T
else:
statfields = ["slope", "intercept", "r", "p", "stderr"]
if not keep_stats:
df = do_stats(df)
if type(df) == bool:
if df is False:
return False
slopes = df.ix["slope"]
if sort_by == "increase":
df = df[slopes.argsort()[::-1]]
elif sort_by == "decrease":
df = df[slopes.argsort()]
elif sort_by == "static":
df = df[slopes.abs().argsort()]
elif sort_by == "turbulent":
df = df[slopes.abs().argsort()[::-1]]
if remove_above_p:
# the easy way to do it!
df = df.T
df = df[df["p"] <= p]
df = df.T
# remove stats field by default
if not keep_stats:
df = df.drop(statfields, axis=0)
return df
def set_threshold(big_list, threshold, prinf=True, for_keywords=False):
if type(threshold) == str:
if threshold.startswith("l"):
denominator = 10000
if threshold.startswith("m"):
denominator = 5000
if threshold.startswith("h"):
denominator = 2500
if type(big_list) == pandas.core.frame.DataFrame:
tot = big_list.sum().sum()
if type(big_list) == pandas.core.series.Series:
tot = big_list.sum()
the_threshold = float(tot) / float(denominator)
# if for_keywords:
# the_threshold = the_threshold / 2
else:
the_threshold = threshold
if prinf:
print("Threshold: %d\n" % the_threshold)
return the_threshold
# copy dataframe to be very safe
df = dataframe1.copy()
# make cols into strings
try:
df.columns = [str(c) for c in list(df.columns)]
except:
pass
if operation is None:
operation = "None"
# do concordance work
if return_conc:
if just_entries:
if type(just_entries) == int:
just_entries = [just_entries]
if type(just_entries) == str:
df = df[df["m"].str.contains(just_entries)]
if type(just_entries) == list:
if all(type(e) == str for e in just_entries):
mp = df["m"].map(lambda x: x in just_entries)
df = df[mp]
else:
df = df.ix[just_entries]
if skip_entries:
if type(skip_entries) == int:
skip_entries = [skip_entries]
if type(skip_entries) == str:
df = df[~df["m"].str.contains(skip_entries)]
if type(skip_entries) == list:
if all(type(e) == str for e in skip_entries):
mp = df["m"].map(lambda x: x not in skip_entries)
df = df[mp]
else:
df = df.drop(skip_entries, axis=0)
if just_subcorpora:
if type(just_subcorpora) == int:
just_subcorpora = [just_subcorpora]
if type(just_subcorpora) == str:
df = df[df["c"].str.contains(just_subcorpora)]
if type(just_subcorpora) == list:
if all(type(e) == str for e in just_subcorpora):
mp = df["c"].map(lambda x: x in just_subcorpora)
df = df[mp]
else:
df = df.ix[just_subcorpora]
if skip_subcorpora:
if type(skip_subcorpora) == int:
skip_subcorpora = [skip_subcorpora]
if type(skip_subcorpora) == str:
df = df[~df["c"].str.contains(skip_subcorpora)]
if type(skip_subcorpora) == list:
if all(type(e) == str for e in skip_subcorpora):
mp = df["c"].map(lambda x: x not in skip_subcorpora)
df = df[mp]
else:
df = df.drop(skip_subcorpora, axis=0)
return Concordance(df)
if print_info:
print("\n***Processing results***\n========================\n")
df1_istotals = False
if type(df) == pandas.core.series.Series:
df1_istotals = True
df = pandas.DataFrame(df)
# if just a single result
else:
df = pandas.DataFrame(df)
if operation.startswith("k"):
if sort_by is False:
if not df1_istotals:
sort_by = "turbulent"
if df1_istotals:
df = df.T
# figure out if there's a second list
# copy and remove totals if there is
single_totals = True
using_totals = False
outputmode = False
if denominator is not False and type(denominator) != str:
df2 = denominator.copy()
using_totals = True
if type(df2) == pandas.core.frame.DataFrame:
if len(df2.columns) > 1:
single_totals = False
else:
df2 = pandas.Series(df2)
if operation == "d":
df2 = df2.sum(axis=1)
single_totals = True
elif type(df2) == pandas.core.series.Series:
single_totals = True
# if operation == 'k':
# raise ValueError('Keywording requires a DataFrame for denominator. Use "self"?')
else:
raise ValueError("Denominator not recognised.")
else:
if operation in ["k", "d", "a", "%", "/", "*", "-", "+"]:
denominator = "self"
if denominator == "self":
outputmode = True
if operation.startswith("a") or operation.startswith("A"):
if list(df.columns)[0] != "0" and list(df.columns)[0] != 0:
df = df.T
if using_totals:
if not single_totals:
df2 = df2.T
if projection:
# projection shouldn't do anything when working with '%', remember.
df = projector(df, projection)
if using_totals:
df2 = projector(df2, projection)
if spelling:
df = convert_spell(df, convert_to=spelling)
df = merge_duplicates(df, print_info=False)
if not single_totals:
df2 = convert_spell(df2, convert_to=spelling, print_info=False)
df2 = merge_duplicates(df2, print_info=False)
if not df1_istotals:
sort_by = "total"
if replace_names:
df = name_replacer(df, replace_names)
df = merge_duplicates(df)
if not single_totals:
df2 = name_replacer(df2, print_info=False)
df2 = merge_duplicates(df2, print_info=False)
if not sort_by:
sort_by = "total"
# remove old stats if they're there:
statfields = ["slope", "intercept", "r", "p", "stderr"]
try:
df = df.drop(statfields, axis=0)
except:
pass
if using_totals:
try:
df2 = df2.drop(statfields, axis=0)
except:
pass
# remove totals and tkinter order
for name, ax in zip(["Total"] * 2 + ["tkintertable-order"] * 2, [0, 1, 0, 1]):
if name == "Total" and df1_istotals:
continue
try:
df = df.drop(name, axis=ax, errors="ignore")
except:
pass
for name, ax in zip(["Total"] * 2 + ["tkintertable-order"] * 2, [0, 1, 0, 1]):
if name == "Total" and single_totals:
continue
try:
df2 = df2.drop(name, axis=ax, errors="ignore")
except:
pass
# merging: make dicts if they aren't already, so we can iterate
if merge_entries:
if type(merge_entries) != list:
if type(merge_entries) == str or type(merge_entries) == str:
merge_entries = {newname: merge_entries}
# for newname, criteria
for name, the_input in sorted(merge_entries.items()):
the_newname = newname_getter(df, parse_input(df, the_input), newname=name, prinf=print_info)
df = merge_these_entries(df, parse_input(df, the_input), the_newname, prinf=print_info)
if not single_totals:
df2 = merge_these_entries(df2, parse_input(df2, the_input), the_newname, prinf=False)
else:
for i in merge_entries:
the_newname = newname_getter(df, parse_input(df, merge_entries), newname=newname, prinf=print_info)
df = merge_these_entries(df, parse_input(df, merge_entries), the_newname, prinf=print_info)
if not single_totals:
df2 = merge_these_entries(df2, parse_input(df2, merge_entries), the_newname, prinf=False)
if merge_subcorpora:
if type(merge_subcorpora) != dict:
if type(merge_subcorpora) == list:
if type(merge_subcorpora[0]) == tuple:
merge_subcorpora = {x: y for x, y in merge_subcorpora}
elif type(merge_subcorpora[0]) == str or type(merge_subcorpora[0]) == str:
merge_subcorpora = {new_subcorpus_name: [x for x in merge_subcorpora]}
elif type(merge_subcorpora[0]) == int:
merge_subcorpora = {new_subcorpus_name: [str(x) for x in merge_subcorpora]}
else:
merge_subcorpora = {new_subcorpus_name: merge_subcorpora}
for name, the_input in sorted(merge_subcorpora.items()):
the_newname = newname_getter(
df.T, parse_input(df.T, the_input), newname=name, merging_subcorpora=True, prinf=print_info
)
df = merge_these_entries(
df.T, parse_input(df.T, the_input), the_newname, merging="subcorpora", prinf=print_info
).T
if using_totals:
df2 = merge_these_entries(
df2.T, parse_input(df2.T, the_input), the_newname, merging="subcorpora", prinf=False
).T
if just_subcorpora:
df = just_these_subcorpora(df, just_subcorpora, prinf=print_info)
if using_totals:
df2 = just_these_subcorpora(df2, just_subcorpora, prinf=False)
if skip_subcorpora:
df = skip_these_subcorpora(df, skip_subcorpora, prinf=print_info)
if using_totals:
df2 = skip_these_subcorpora(df2, skip_subcorpora, prinf=False)
if span_subcorpora:
df = span_these_subcorpora(df, span_subcorpora, prinf=print_info)
if using_totals:
df2 = span_these_subcorpora(df2, span_subcorpora, prinf=False)
if just_entries:
df = just_these_entries(df, parse_input(df, just_entries), prinf=print_info)
if not single_totals:
df2 = just_these_entries(df2, parse_input(df2, just_entries), prinf=False)
if skip_entries:
df = skip_these_entries(df, parse_input(df, skip_entries), prinf=print_info)
if not single_totals:
df2 = skip_these_entries(df2, parse_input(df2, skip_entries), prinf=False)
# drop infinites and nans
if operation != "d":
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
# make just_totals as dataframe
just_one_total_number = False
if just_totals:
df = pd.DataFrame(df.sum(), columns=["Combined total"])
if using_totals:
if not single_totals:
df2 = pd.DataFrame(df2.sum(), columns=["Combined total"])
else:
just_one_total_number = True
df2 = df2.sum()
tots = df.sum(axis=1)
if using_totals or outputmode:
if not operation.startswith("k"):
the_threshold = 0
# set a threshold if just_totals
if outputmode is True:
df2 = df.T.sum()
if not just_totals:
df2.name = "Total"
else:
df2.name = "Combined total"
using_totals = True
single_totals = True
if just_totals:
if not single_totals:
the_threshold = set_threshold(df2, threshold, prinf=print_info)
if operation == "d":
the_threshold = set_threshold(df2, threshold, prinf=print_info)
df, tots = combiney(df, df2, operation=operation, threshold=the_threshold, prinf=print_info)
# if doing keywording...
if operation.startswith("k"):
from keys import keywords
# allow saved dicts to be df2, etc
try:
if denominator == "self":
df2 = df.copy()
except TypeError:
pass
if type(denominator) == str:
if denominator != "self":
df2 = denominator
else:
the_threshold = False
df = keywords(
df,
df2,
selfdrop=selfdrop,
threshold=threshold,
printstatus=print_info,
editing=True,
calc_all=calc_all,
**kwargs
)
# eh?
df = df.T
# drop infinites and nans
if operation != "d":
df = df.replace([np.inf, -np.inf], np.nan)
df = df.fillna(0.0)
# resort data
if sort_by or keep_stats:
df = resort(df, keep_stats=keep_stats, sort_by=sort_by)
if type(df) == bool:
if df is False:
return "linregress"
if keep_top:
if not just_totals:
df = df[list(df.columns)[:keep_top]]
else:
df = df.head(keep_top)
if just_totals:
# turn just_totals into series:
df = pd.Series(df["Combined total"], name="Combined total")
if df1_istotals:
if operation.startswith("k"):
try:
df = pd.Series(df.ix[dataframe1.name])
df.name = "%s: keyness" % df.name
except:
df = df.iloc[0, :]
df.name = "keyness" % df.name
# generate totals branch if not percentage results:
# fix me
if df1_istotals or operation.startswith("k"):
if not just_totals:
try:
total = pd.Series(df["Total"], name="Total")
except:
pass
total = "none"
# total = df.copy()
else:
total = "none"
else:
# might be wrong if using division or something...
try:
total = df.T.sum(axis=1)
except:
total = "none"
if type(tots) != pandas.core.frame.DataFrame and type(tots) != pandas.core.series.Series:
total = df.sum(axis=1)
else:
total = tots
if type(df) == pandas.core.frame.DataFrame:
datatype = df.ix[0].dtype
else:
datatype = df.dtype
# TURN INT COL NAMES INTO STR
try:
df.results.columns = [str(d) for d in list(df.results.columns)]
except:
pass
def add_tkt_index(df):
if type(df) != pandas.core.series.Series:
df = df.T
df = df.drop("tkintertable-order", errors="ignore", axis=0)
df = df.drop("tkintertable-order", errors="ignore", axis=1)
df["tkintertable-order"] = pd.Series(
[index for index, data in enumerate(list(df.index))], index=list(df.index)
)
df = df.T
return df
# while tkintertable can't sort rows
from tests import check_t_kinter
tk = check_t_kinter()
if tk:
df = add_tkt_index(df)
if kwargs.get("df1_always_df"):
if type(df) == pandas.core.series.Series:
df = pandas.DataFrame(df)
# outputnames = collections.namedtuple('edited_interrogation', ['query', 'results', 'totals'])
# output = outputnames(the_options, df, total)
# delete non-appearing conc lines
if interrogation.__dict__.get("concordance", None) is None:
lns = None
else:
col_crit = interrogation.concordance["m"].map(lambda x: x in list(df.columns))
ind_crit = interrogation.concordance["c"].map(lambda x: x in list(df.index))
lns = interrogation.concordance[col_crit]
lns = lns.loc[ind_crit]
lns = Concordance(lns)
output = Interrogation(results=df, totals=total, query=locs, concordance=lns)
# print '\nResult (sample)\n'
if print_info:
# if merge_entries or merge_subcorpora or span_subcorpora or just_subcorpora or \
# just_entries or skip_entries or skip_subcorpora or printed_th or projection:
print("***Done!***\n========================\n")
# print df.head().T
# print ''
if operation.startswith("k") or just_totals or df1_istotals:
pd.set_option("display.max_rows", 30)
else:
pd.set_option("display.max_rows", 15)
pd.set_option("display.max_columns", 8)
pd.set_option("max_colwidth", 70)
pd.set_option("display.width", 800)
pd.set_option("expand_frame_repr", False)
pd.set_option("display.float_format", lambda x: "%.2f" % x)
return output
def plotter(df,
title=False,
kind='line',
x_label=None,
y_label=None,
style='ggplot',
figsize=(8, 4),
save=False,
legend_pos='best',
reverse_legend='guess',
num_to_plot=7,
tex='try',
colours='Accent',
cumulative=False,
pie_legend=True,
partial_pie=False,
show_totals=False,
transparent=False,
output_format='png',
interactive=False,
black_and_white=False,
show_p_val=False,
indices=False,
transpose=False,
**kwargs):
"""Visualise corpus interrogations.
:param title: A title for the plot
:type title: str
:param df: Data to be plotted
:type df: pandas.core.frame.DataFrame
:param x_label: A label for the x axis
:type x_label: str
:param y_label: A label for the y axis
:type y_label: str
:param kind: The kind of chart to make
:type kind: str ('line'/'bar'/'barh'/'pie'/'area')
:param style: Visual theme of plot
:type style: str ('ggplot'/'bmh'/'fivethirtyeight'/'seaborn-talk'/etc)
:param figsize: Size of plot
:type figsize: tuple (int, int)
:param save: If bool, save with *title* as name; if str, use str as name
:type save: bool/str
:param legend_pos: Where to place legend
:type legend_pos: str ('upper right'/'outside right'/etc)
:param reverse_legend: Reverse the order of the legend
:type reverse_legend: bool
:param num_to_plot: How many columns to plot
:type num_to_plot: int/'all'
:param tex: Use TeX to draw plot text
:type tex: bool
:param colours: Colourmap for lines/bars/slices
:type colours: str
:param cumulative: Plot values cumulatively
:type cumulative: bool
:param pie_legend: Show a legend for pie chart
:type pie_legend: bool
:param partial_pie: Allow plotting of pie slices only
:type partial_pie: bool
:param show_totals: Print sums in plot where possible
:type show_totals: str -- 'legend'/'plot'/'both'
:param transparent: Transparent .png background
:type transparent: bool
:param output_format: File format for saved image
:type output_format: str -- 'png'/'pdf'
:param black_and_white: Create black and white line styles
:type black_and_white: bool
:param show_p_val: Attempt to print p values in legend if contained in df
:type show_p_val: bool
:param indices: To use when plotting "distance from root"
:type indices: bool
:param stacked: When making bar chart, stack bars on top of one another
:type stacked: str
:param filled: For area and bar charts, make every column sum to 100
:type filled: str
:param legend: Show a legend
:type legend: bool
:param rot: Rotate x axis ticks by *rot* degrees
:type rot: int
:param subplots: Plot each column separately
:type subplots: bool
:param layout: Grid shape to use when *subplots* is True
:type layout: tuple -- (int, int)
:param interactive: Experimental interactive options
:type interactive: list -- [1, 2, 3]
:returns: matplotlib figure
"""
import corpkit
import os
try:
from IPython.utils.shimmodule import ShimWarning
import warnings
warnings.simplefilter('ignore', ShimWarning)
except:
pass
import matplotlib as mpl
from matplotlib import rc
# prefer seaborn plotting
try:
import seaborn as sns
except:
pass
if interactive:
import matplotlib.pyplot as plt, mpld3
else:
import matplotlib.pyplot as plt
import pandas
from pandas import DataFrame
import numpy
from time import localtime, strftime
from tests import check_pytex, check_spider, check_t_kinter
if interactive:
import mpld3
import collections
from mpld3 import plugins, utils
from plugins import InteractiveLegendPlugin, HighlightLines
# check what environment we're in
tk = check_t_kinter()
running_python_tex = check_pytex()
running_spider = check_spider()
if not title:
title = ''
def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
"""remove extreme values from colourmap --- no pure white"""
import matplotlib.colors as colors
import numpy as np
new_cmap = colors.LinearSegmentedColormap.from_list(
'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name,
a=minval,
b=maxval),
cmap(np.linspace(minval, maxval, n)))
return new_cmap
def get_savename(imagefolder, save=False, title=False, ext='png'):
"""Come up with the savename for the image."""
import os
def urlify(s):
"Turn title into filename"
import re
s = s.lower()
s = re.sub(r"[^\w\s-]", '', s)
s = re.sub(r"\s+", '-', s)
s = re.sub(r"-(textbf|emph|textsc|textit)", '-', s)
return s
# name as
if not ext.startswith('.'):
ext = '.' + ext
if type(save) == str:
savename = os.path.join(imagefolder, (urlify(save) + ext))
#this 'else' is redundant now that title is obligatory
else:
if title:
filename = urlify(title) + ext
savename = os.path.join(imagefolder, filename)
# remove duplicated ext
if savename.endswith('%s%s' % (ext, ext)):
savename = savename.replace('%s%s' % (ext, ext), ext, 1)
return savename
def rename_data_with_total(dataframe,
was_series=False,
using_tex=False,
absolutes=True):
"""adds totals (abs, rel, keyness) to entry name strings"""
if was_series:
where_the_words_are = dataframe.index
else:
where_the_words_are = dataframe.columns
the_labs = []
for w in list(where_the_words_are):
if not absolutes:
if was_series:
perc = dataframe.T[w][0]
else:
the_labs.append(w)
continue
if using_tex:
the_labs.append('%s (%.2f\%%)' % (w, perc))
else:
the_labs.append('%s (%.2f %%)' % (w, perc))
else:
if was_series:
score = dataframe.T[w].sum()
else:
score = dataframe[w].sum()
if using_tex:
the_labs.append('%s (n=%d)' % (w, score))
else:
the_labs.append('%s (n=%d)' % (w, score))
if not was_series:
dataframe.columns = the_labs
else:
vals = list(dataframe[list(dataframe.columns)[0]].values)
dataframe = pandas.DataFrame(vals, index=the_labs)
dataframe.columns = ['Total']
return dataframe
def auto_explode(dataframe, input, was_series=False, num_to_plot=7):
"""give me a list of strings and i'll output explode option"""
output = [0 for s in range(num_to_plot)]
if was_series:
l = list(dataframe.index)
else:
l = list(dataframe.columns)
if type(input) == str or type(input) == int:
input = [input]
if type(input) == list:
for i in input:
if type(i) == str:
index = l.index(i)
else:
index = i
output[index] = 0.1
return output
# check if we're doing subplots
sbplt = False
if 'subplots' in kwargs:
if kwargs['subplots'] is True:
sbplt = True
kwargs['subplots'] = sbplt
if colours is True:
colours = 'Paired'
# todo: get this dynamically instead.
styles = [
'dark_background', 'bmh', 'grayscale', 'ggplot', 'fivethirtyeight',
'matplotlib', False, 'mpl-white'
]
#if style not in styles:
#raise ValueError('Style %s not found. Use %s' % (str(style), ', '.join(styles)))
if style == 'mpl-white':
try:
sns.set_style("whitegrid")
except:
pass
style = 'matplotlib'
if style is not False and style.startswith('seaborn'):
colours = False
# use 'draggable = True' to make a draggable legend
dragmode = kwargs.pop('draggable', False)
if kwargs.get('savepath'):
mpl.rcParams['savefig.directory'] = kwargs.get('savepath')
kwargs.pop('savepath', None)
mpl.rcParams['savefig.bbox'] = 'tight'
mpl.rcParams.update({'figure.autolayout': True})
# try to use tex
# TO DO:
# make some font kwargs here
using_tex = False
mpl.rcParams['font.family'] = 'sans-serif'
mpl.rcParams['text.latex.unicode'] = True
if tex == 'try' or tex is True:
try:
rc('text', usetex=True)
rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
using_tex = True
except:
matplotlib.rc('font', family='sans-serif')
matplotlib.rc('font', serif='Helvetica Neue')
matplotlib.rc('text', usetex='false')
rc('text', usetex=False)
else:
rc('text', usetex=False)
if interactive:
using_tex = False
if show_totals is False:
show_totals = 'none'
# find out what kind of plot we're making, and enable
# or disable interactive values if need be
kwargs['kind'] = kind.lower()
if interactive:
if kwargs['kind'].startswith('bar'):
interactive_types = [3]
elif kwargs['kind'] == 'area':
interactive_types = [2, 3]
elif kwargs['kind'] == 'line':
interactive_types = [2, 3]
elif kwargs['kind'] == 'pie':
interactive_types = None
warnings.warn(
'Interactive plotting not yet available for pie plots.')
else:
interactive_types = [None]
if interactive is False:
interactive_types = [None]
# find out if pie mode, add autopct format
piemode = False
if kind == 'pie':
piemode = True
# always the best spot for pie
#if legend_pos == 'best':
#legend_pos = 'lower left'
if show_totals.endswith('plot') or show_totals.endswith('both'):
kwargs['pctdistance'] = 0.6
if using_tex:
kwargs['autopct'] = r'%1.1f\%%'
else:
kwargs['autopct'] = '%1.1f%%'
# copy data, make series into df
dataframe = df.copy()
was_series = False
if type(dataframe) == pandas.core.series.Series:
was_series = True
if not cumulative:
dataframe = DataFrame(dataframe)
else:
dataframe = DataFrame(dataframe.cumsum())
else:
# don't know if this is much good.
if transpose:
dataframe = dataframe.T
if cumulative:
dataframe = DataFrame(dataframe.cumsum())
if len(list(dataframe.columns)) == 1:
was_series = True
# attempt to convert x axis to ints:
try:
dataframe.index = [int(i) for i in list(dataframe.index)]
except:
pass
# remove totals and tkinter order
if not was_series and not all(x.lower() == 'total'
for x in list(dataframe.columns)):
for name, ax in zip(['Total'] * 2 + ['tkintertable-order'] * 2,
[0, 1, 0, 1]):
try:
dataframe = dataframe.drop(name, axis=ax, errors='ignore')
except:
pass
else:
dataframe = dataframe.drop('tkintertable-order', errors='ignore')
dataframe = dataframe.drop('tkintertable-order',
axis=1,
errors='ignore')
# look at columns to see if all can be ints, in which case, set up figure
# for depnumming
if not was_series:
if indices == 'guess':
def isint(x):
try:
a = float(x)
b = int(a)
except ValueError or OverflowError:
return False
else:
return a == b
if all([isint(x) is True for x in list(dataframe.columns)]):
indices = True
else:
indices = False
# if depnumming, plot all, transpose, and rename axes
if indices is True:
num_to_plot = 'all'
dataframe = dataframe.T
if y_label is None:
y_label = 'Percentage of all matches'
if x_label is None:
x_label = ''
# set backend?
output_formats = [
'svgz', 'ps', 'emf', 'rgba', 'raw', 'pdf', 'svg', 'eps', 'png', 'pgf'
]
if output_format not in output_formats:
raise ValueError('%s output format not recognised. Must be: %s' %
(output_format, ', '.join(output_formats)))
# don't know if these are necessary
if 'pdf' in output_format:
plt.switch_backend(output_format)
if 'pgf' in output_format:
plt.switch_backend(output_format)
if num_to_plot == 'all':
if was_series:
if not piemode:
num_to_plot = len(dataframe)
else:
num_to_plot = len(dataframe)
else:
if not piemode:
num_to_plot = len(list(dataframe.columns))
else:
num_to_plot = len(dataframe.index)
# explode pie, or remove if not piemode
if piemode and not sbplt and kwargs.get('explode'):
kwargs['explode'] = auto_explode(dataframe,
kwargs['explode'],
was_series=was_series,
num_to_plot=num_to_plot)
else:
kwargs.pop('explode', None)
legend = kwargs.get('legend', True)
#cut data short
plotting_a_totals_column = False
if was_series:
if list(dataframe.columns)[0] != 'Total':
try:
can_be_ints = [int(x) for x in list(dataframe.index)]
num_to_plot = len(dataframe)
except:
dataframe = dataframe[:num_to_plot]
elif list(dataframe.columns)[0] == 'Total':
plotting_a_totals_column = True
if not 'legend' in kwargs:
legend = False
num_to_plot = len(dataframe)
else:
dataframe = dataframe.T.head(num_to_plot).T
# remove stats fields, put p in entry text, etc.
statfields = ['slope', 'intercept', 'r', 'p', 'stderr']
try:
dataframe = dataframe.drop(statfields, axis=1, errors='ignore')
except:
pass
try:
dataframe.ix['p']
there_are_p_vals = True
except:
there_are_p_vals = False
if show_p_val:
if there_are_p_vals:
newnames = []
for col in list(dataframe.columns):
pval = dataframe[col]['p']
def p_string_formatter(val):
if val < 0.001:
if not using_tex:
return 'p < 0.001'
else:
return r'p $<$ 0.001'
else:
return 'p = %s' % format(val, '.3f')
pstr = p_string_formatter(pval)
newname = '%s (%s)' % (col, pstr)
newnames.append(newname)
dataframe.columns = newnames
dataframe.drop(statfields, axis=0, inplace=True, errors='ignore')
else:
warnings.warn(
'No p-values calculated to show.\n\nUse sort_by and keep_stats in editor() to generate these values.'
)
else:
if there_are_p_vals:
dataframe.drop(statfields, axis=0, inplace=True, errors='ignore')
# make and set y label
absolutes = True
if type(dataframe) == pandas.core.frame.DataFrame:
try:
if not all([s.is_integer() for s in dataframe.iloc[0, :].values]):
absolutes = False
except:
pass
else:
if not all([s.is_integer() for s in dataframe.values]):
absolutes = False
# use colormap if need be:
if num_to_plot > 0:
if not was_series:
if kind in ['pie', 'line', 'area']:
if colours:
if not plotting_a_totals_column:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
#else:
if colours:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
if piemode:
if num_to_plot > 0:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
else:
if num_to_plot > 0:
if colours == 'Default':
colours = 'Paired'
kwargs['colormap'] = colours
# multicoloured bar charts
if colours:
if kind.startswith('bar'):
if len(list(dataframe.columns)) == 1:
if not black_and_white:
import numpy as np
the_range = np.linspace(0, 1, num_to_plot)
middle = len(the_range) / 2
cmap = plt.get_cmap(colours)
kwargs['color'] = [cmap(n) for n in the_range][middle]
# make a bar width ... ? ...
#kwargs['width'] = (figsize[0] / float(num_to_plot)) / 1.5
# reversing legend option
if reverse_legend is True:
rev_leg = True
elif reverse_legend is False:
rev_leg = False
# show legend or don't, guess whether to reverse based on kind
if kind in ['bar', 'barh', 'area', 'line', 'pie']:
if was_series:
legend = False
if kind == 'pie':
if pie_legend:
legend = True
else:
legend = False
if kind in ['barh', 'area']:
if reverse_legend == 'guess':
rev_leg = True
if not 'rev_leg' in locals():
rev_leg = False
# the default legend placement
if legend_pos is True:
legend_pos = 'best'
# cut dataframe if just_totals
try:
tst = dataframe['Combined total']
dataframe = dataframe.head(num_to_plot)
except:
pass
# rotate automatically
if 'rot' not in kwargs:
if not was_series:
xvals = [str(i) for i in list(dataframe.index)[:num_to_plot]]
#if 'kind' in kwargs:
#if kwargs['kind'] in ['barh', 'area']:
#xvals = [str(i) for i in list(dataframe.columns)[:num_to_plot]]
else:
xvals = [str(i) for i in list(dataframe.columns)[:num_to_plot]]
if len(max(xvals, key=len)) > 6:
if not piemode:
kwargs['rot'] = 45
else:
kwargs['rot'] = False
# no title for subplots because ugly,
if title and not sbplt:
kwargs['title'] = title
# no interactive subplots yet:
if sbplt and interactive:
import warnings
interactive = False
warnings.warn('No interactive subplots yet, sorry.')
return
# not using pandas for labels or legend anymore.
#kwargs['labels'] = None
#kwargs['legend'] = False
if legend:
if num_to_plot > 6:
if not kwargs.get('ncol'):
kwargs['ncol'] = num_to_plot / 7
# kwarg options go in leg_options
leg_options = {
'framealpha': .8,
'shadow': kwargs.get('shadow', False),
'ncol': kwargs.pop('ncol', 1)
}
# determine legend position based on this dict
if legend_pos:
possible = {
'best': 0,
'upper right': 1,
'upper left': 2,
'lower left': 3,
'lower right': 4,
'right': 5,
'center left': 6,
'center right': 7,
'lower center': 8,
'upper center': 9,
'center': 10,
'o r': 2,
'outside right': 2,
'outside upper right': 2,
'outside center right': 'center left',
'outside lower right': 'lower left'
}
if type(legend_pos) == int:
the_loc = legend_pos
elif type(legend_pos) == str:
try:
the_loc = possible[legend_pos]
except KeyError:
raise KeyError(
'legend_pos value must be one of:\n%s\n or an int between 0-10.'
% ', '.join(list(possible.keys())))
leg_options['loc'] = the_loc
#weirdness needed for outside plot
if legend_pos in ['o r', 'outside right', 'outside upper right']:
leg_options['bbox_to_anchor'] = (1.02, 1)
if legend_pos == 'outside center right':
leg_options['bbox_to_anchor'] = (1.02, 0.5)
if legend_pos == 'outside lower right':
leg_options['loc'] == 'upper right'
leg_options['bbox_to_anchor'] = (0.5, 0.5)
# a bit of distance between legend and plot for outside legends
if type(legend_pos) == str:
if legend_pos.startswith('o'):
leg_options['borderaxespad'] = 1
if not piemode:
if show_totals.endswith('both') or show_totals.endswith('legend'):
dataframe = rename_data_with_total(dataframe,
was_series=was_series,
using_tex=using_tex,
absolutes=absolutes)
else:
if pie_legend:
if show_totals.endswith('both') or show_totals.endswith('legend'):
dataframe = rename_data_with_total(dataframe,
was_series=was_series,
using_tex=using_tex,
absolutes=absolutes)
if piemode:
if partial_pie:
dataframe = dataframe / 100.0
# some pie things
if piemode:
if not sbplt:
kwargs['y'] = list(dataframe.columns)[0]
if pie_legend:
kwargs['legend'] = False
if was_series:
leg_options['labels'] = list(dataframe.index)
else:
leg_options['labels'] = list(dataframe.columns)
else:
if pie_legend:
kwargs['legend'] = False
if was_series:
leg_options['labels'] = list(dataframe.index)
else:
leg_options['labels'] = list(dataframe.index)
def filler(df):
pby = df.T.copy()
for i in list(pby.columns):
tot = pby[i].sum()
pby[i] = pby[i] * 100.0 / tot
return pby.T
areamode = False
if kind == 'area':
areamode = True
if legend is False:
kwargs['legend'] = False
# line highlighting option for interactive!
if interactive:
if 2 in interactive_types:
if kind == 'line':
kwargs['marker'] = ','
if not piemode:
kwargs['alpha'] = 0.1
# convert dates --- works only in my current case!
if plotting_a_totals_column or not was_series:
try:
can_it_be_int = int(list(dataframe.index)[0])
can_be_int = True
except:
can_be_int = False
if can_be_int:
if 1500 < int(list(dataframe.index)[0]):
if 2050 > int(list(dataframe.index)[0]):
n = pandas.PeriodIndex([d for d in list(dataframe.index)],
freq='A')
dataframe = dataframe.set_index(n)
if kwargs.get('filled'):
if areamode or kind.startswith('bar'):
dataframe = filler(dataframe)
kwargs.pop('filled', None)
MARKERSIZE = 4
COLORMAP = {
0: {
'marker': None,
'dash': (None, None)
},
1: {
'marker': None,
'dash': [5, 5]
},
2: {
'marker': "o",
'dash': (None, None)
},
3: {
'marker': None,
'dash': [1, 3]
},
4: {
'marker': "s",
'dash': [5, 2, 5, 2, 5, 10]
},
5: {
'marker': None,
'dash': [5, 3, 1, 2, 1, 10]
},
6: {
'marker': 'o',
'dash': (None, None)
},
7: {
'marker': None,
'dash': [5, 3, 1, 3]
},
8: {
'marker': "1",
'dash': [1, 3]
},
9: {
'marker': "*",
'dash': [5, 5]
},
10: {
'marker': "2",
'dash': [5, 2, 5, 2, 5, 10]
},
11: {
'marker': "s",
'dash': (None, None)
}
}
HATCHES = {
0: {
'color': '#dfdfdf',
'hatch': "/"
},
1: {
'color': '#6f6f6f',
'hatch': "\\"
},
2: {
'color': 'b',
'hatch': "|"
},
3: {
'color': '#dfdfdf',
'hatch': "-"
},
4: {
'color': '#6f6f6f',
'hatch': "+"
},
5: {
'color': 'b',
'hatch': "x"
}
}
if black_and_white:
if kind == 'line':
kwargs['linewidth'] = 1
cmap = plt.get_cmap('Greys')
new_cmap = truncate_colormap(cmap, 0.25, 0.95)
if kind == 'bar':
# darker if just one entry
if len(dataframe.columns) == 1:
new_cmap = truncate_colormap(cmap, 0.70, 0.90)
kwargs['colormap'] = new_cmap
class dummy_context_mgr():
"""a fake context for plotting without style
perhaps made obsolete by 'classic' style in new mpl"""
def __enter__(self):
return None
def __exit__(self, one, two, three):
return False
with plt.style.context(
(style)) if style != 'matplotlib' else dummy_context_mgr():
if not sbplt:
# check if negative values, no stacked if so
if areamode:
kwargs['legend'] = False
if dataframe.applymap(lambda x: x < 0.0).any().any():
kwargs['stacked'] = False
rev_leg = False
ax = dataframe.plot(figsize=figsize, **kwargs)
if areamode:
handles, labels = plt.gca().get_legend_handles_labels()
del handles
del labels
else:
plt.gcf().set_tight_layout(False)
if not piemode:
ax = dataframe.plot(figsize=figsize, **kwargs)
else:
ax = dataframe.plot(figsize=figsize, **kwargs)
handles, labels = plt.gca().get_legend_handles_labels()
plt.legend(handles,
labels,
loc=leg_options['loc'],
bbox_to_anchor=(0, -0.1, 1, 1),
bbox_transform=plt.gcf().transFigure)
# this line allows layouts with missing plots
# i.e. layout = (5, 2) with only nine plots
plt.gcf().set_tight_layout(False)
if 'rot' in kwargs:
if kwargs['rot'] != 0 and kwargs['rot'] != 90:
labels = [item.get_text() for item in ax.get_xticklabels()]
ax.set_xticklabels(labels, rotation=kwargs['rot'], ha='right')
if transparent:
plt.gcf().patch.set_facecolor('white')
plt.gcf().patch.set_alpha(0)
if black_and_white:
if kind == 'line':
# white background
# change everything to black and white with interesting dashes and markers
c = 0
for line in ax.get_lines():
line.set_color('black')
#line.set_width(1)
line.set_dashes(COLORMAP[c]['dash'])
line.set_marker(COLORMAP[c]['marker'])
line.set_markersize(MARKERSIZE)
c += 1
if c == len(list(COLORMAP.keys())):
c = 0
# draw legend with proper placement etc
if legend:
if not piemode and not sbplt:
if 3 not in interactive_types:
handles, labels = plt.gca().get_legend_handles_labels()
# area doubles the handles and labels. this removes half:
if areamode:
handles = handles[-len(handles) / 2:]
labels = labels[-len(labels) / 2:]
if rev_leg:
handles = handles[::-1]
labels = labels[::-1]
lgd = plt.legend(handles, labels, **leg_options)
if interactive:
# 1 = highlight lines
# 2 = line labels
# 3 = legend switches
ax = plt.gca()
# fails for piemode
lines = ax.lines
handles, labels = plt.gca().get_legend_handles_labels()
if 1 in interactive_types:
plugins.connect(plt.gcf(), HighlightLines(lines))
if 3 in interactive_types:
plugins.connect(
plt.gcf(),
InteractiveLegendPlugin(lines, labels, alpha_unsel=0.0))
for i, l in enumerate(lines):
y_vals = l.get_ydata()
x_vals = l.get_xdata()
x_vals = [str(x) for x in x_vals]
if absolutes:
ls = [
'%s (%s: %d)' % (labels[i], x_val, y_val)
for x_val, y_val in zip(x_vals, y_vals)
]
else:
ls = [
'%s (%s: %.2f%%)' % (labels[i], x_val, y_val)
for x_val, y_val in zip(x_vals, y_vals)
]
if 2 in interactive_types:
#if 'kind' in kwargs and kwargs['kind'] == 'area':
tooltip_line = mpld3.plugins.LineLabelTooltip(
lines[i], labels[i])
mpld3.plugins.connect(plt.gcf(), tooltip_line)
#else:
if kind == 'line':
tooltip_point = mpld3.plugins.PointLabelTooltip(l,
labels=ls)
mpld3.plugins.connect(plt.gcf(), tooltip_point)
if piemode:
if not sbplt:
plt.axis('equal')
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# add x label
# this could be revised now!
# if time series period, it's year for now
if type(dataframe.index) == pandas.tseries.period.PeriodIndex:
x_label = 'Year'
if x_label is not False:
if type(x_label) == str:
plt.xlabel(x_label)
else:
check_x_axis = list(
dataframe.index
)[0] # get first entry# get second entry of first entry (year, count)
try:
if type(dataframe.index) == pandas.tseries.period.PeriodIndex:
x_label = 'Year'
check_x_axis = int(check_x_axis)
if 1500 < check_x_axis < 2050:
x_label = 'Year'
else:
x_label = 'Group'
except:
x_label = 'Group'
if not sbplt:
if not piemode:
plt.xlabel(x_label)
def is_number(s):
"""check if str can be can be made into float/int"""
try:
float(s) # for int, long and float
except ValueError:
try:
complex(s) # for complex
except ValueError:
return False
return True
# for now, always turn off sci notation
from matplotlib.ticker import ScalarFormatter
if type(dataframe.index) != pandas.tseries.period.PeriodIndex:
try:
if all(is_number(s) for s in list(dataframe.index)):
plt.gca().xaxis.set_major_formatter(ScalarFormatter())
except:
pass
try:
if all(is_number(s) for s in list(dataframe.columns)):
plt.gca().yaxis.set_major_formatter(ScalarFormatter())
except:
pass
# y labelling
y_l = False
if not absolutes:
y_l = 'Percentage'
else:
y_l = 'Absolute frequency'
def suplabel(axis,
label,
label_prop=None,
labelpad=5,
ha='center',
va='center'):
''' Add super ylabel or xlabel to the figure
Similar to matplotlib.suptitle
axis - string: "x" or "y"
label - string
label_prop - keyword dictionary for Text
labelpad - padding from the axis (default: 5)
ha - horizontal alignment (default: "center")
va - vertical alignment (default: "center")
'''
fig = plt.gcf()
xmin = []
ymin = []
for ax in fig.axes:
xmin.append(ax.get_position().xmin)
ymin.append(ax.get_position().ymin)
xmin, ymin = min(xmin), min(ymin)
dpi = fig.dpi
if axis.lower() == "y":
rotation = 90.
x = xmin - float(labelpad) / dpi
y = 0.5
elif axis.lower() == 'x':
rotation = 0.
x = 0.5
y = ymin - float(labelpad) / dpi
else:
raise Exception("Unexpected axis: x or y")
if label_prop is None:
label_prop = dict()
plt.gcf().text(x,
y,
label,
rotation=rotation,
transform=fig.transFigure,
ha=ha,
va=va,
**label_prop)
if y_label is not False:
if not sbplt:
if not piemode:
if type(y_label) == str:
plt.ylabel(y_label)
else:
plt.ylabel(y_l)
else:
if type(y_label) == str:
the_y = y_label
else:
the_y = y_l
#suplabel('y', the_y, labelpad = 1.5)
plt.gcf().text(0.04, 0.5, the_y, va='center', rotation='vertical')
#plt.subplots_adjust(left=0.5)
# if not piemode:
# if type(y_label) == str:
# plt.ylabel(y_label)
# else:
# plt.ylabel(y_l)
# hacky: turn legend into subplot titles :)
if sbplt:
# title the big plot
#plt.gca().suptitle(title, fontsize = 16)
#plt.subplots_adjust(top=0.9)
# get all axes
if 'layout' not in kwargs:
axes = [l for index, l in enumerate(ax)]
else:
axes = []
cols = [l for index, l in enumerate(ax)]
for col in cols:
for bit in col:
axes.append(bit)
# set subplot titles
for index, a in enumerate(axes):
try:
titletext = list(dataframe.columns)[index]
except:
pass
a.set_title(titletext)
try:
a.legend_.remove()
except:
pass
# remove axis labels for pie plots
if piemode:
a.axes.get_xaxis().set_visible(False)
a.axes.get_yaxis().set_visible(False)
a.axis('equal')
# show grid
a.grid(b=kwargs.get('grid', False))
kwargs.pop('grid', None)
# add sums to bar graphs and pie graphs
# doubled right now, no matter
if not sbplt:
if kind.startswith('bar'):
width = ax.containers[0][0].get_width()
# show grid
ax.grid(b=kwargs.get('grid', False))
kwargs.pop('grid', None)
if was_series:
the_y_limit = plt.ylim()[1]
if show_totals.endswith('plot') or show_totals.endswith('both'):
# make plot a bit higher if putting these totals on it
plt.ylim([0, the_y_limit * 1.05])
for i, label in enumerate(list(dataframe.index)):
if len(dataframe.ix[label]) == 1:
score = dataframe.ix[label][0]
else:
if absolutes:
score = dataframe.ix[label].sum()
else:
#import warnings
#warnings.warn("It's not possible to determine total percentage from individual percentages.")
continue
if not absolutes:
plt.annotate('%.2f' % score, (i, score),
ha='center',
va='bottom')
else:
plt.annotate(score, (i, score), ha='center', va='bottom')
else:
the_y_limit = plt.ylim()[1]
if show_totals.endswith('plot') or show_totals.endswith('both'):
for i, label in enumerate(list(dataframe.columns)):
if len(dataframe[label]) == 1:
score = dataframe[label][0]
else:
if absolutes:
score = dataframe[label].sum()
else:
#import warnings
#warnings.warn("It's not possible to determine total percentage from individual percentages.")
continue
if not absolutes:
plt.annotate('%.2f' % score, (i, score),
ha='center',
va='bottom')
else:
plt.annotate(score, (i, score), ha='center', va='bottom')
plt.subplots_adjust(left=0.1)
plt.subplots_adjust(bottom=0.18)
if 'layout' not in kwargs:
if not sbplt:
plt.tight_layout()
if save:
import os
if running_python_tex:
imagefolder = '../images'
else:
imagefolder = 'images'
savename = get_savename(imagefolder,
save=save,
title=title,
ext=output_format)
if not os.path.isdir(imagefolder):
os.makedirs(imagefolder)
# save image and get on with our lives
if legend_pos.startswith('o'):
plt.gcf().savefig(savename,
dpi=150,
bbox_extra_artists=(lgd, ),
bbox_inches='tight',
format=output_format)
else:
plt.gcf().savefig(savename, dpi=150, format=output_format)
time = strftime("%H:%M:%S", localtime())
if os.path.isfile(savename):
print('\n' + time + ": " + savename + " created.")
else:
raise ValueError("Error making %s." % savename)
if dragmode:
plt.legend().draggable()
if sbplt:
plt.subplots_adjust(right=.8)
plt.subplots_adjust(left=.1)
if not interactive and not running_python_tex and not running_spider \
and not tk:
plt.gcf().show()
return
elif running_spider or tk:
return plt
if interactive:
plt.subplots_adjust(right=.8)
plt.subplots_adjust(left=.1)
try:
ax.legend_.remove()
except:
pass
return mpld3.display()
