def test_basic_matplotlib():
"""
Based on the demo at: http://matplotlib.org/examples/lines_bars_and_markers/barh_demo.html
Viewed on 4 August 2014
Simple demo of a horizontal bar chart.
"""
import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
# Example data
people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim')
y_pos = np.arange(len(people))
performance = 3 + 10 * np.random.rand(len(people))
error = np.random.rand(len(people))
plt.barh(y_pos, performance, xerr=error, align='center', alpha=0.4)
plt.yticks(y_pos, people)
plt.xlabel('Performance')
plt.title('How fast do you want to go today?')
text = wordgraph.describe(plt, source='matplotlib')
assert text is not None
def setup_location(station_type):
data = testing.getTestROSData()
np.random.seed(0)
loc = Location(data, station_type=station_type, bsIter=10000,
rescol='res', qualcol='qual', useROS=True)
plt.rcdefaults()
return loc
def graph_FWHM_data_range(start_date=datetime.datetime(2015,3,6),
end_date=datetime.datetime(2015,4,15),tenmin=True,
path='/home/douglas/Dropbox (Thacher)/Observatory/Seeing/Data/',
write=True,outpath='./'):
plot_params()
fwhm = get_FWHM_data_range(start_date = start_date, end_date=end_date, path=path, tenmin=tenmin)
# Basic stats
med = np.median(fwhm)
mean = np.mean(fwhm)
fwhm_clip, low, high = sigmaclip(fwhm,low=3,high=3)
meanclip = np.mean(fwhm_clip)
# Get mode using kernel density estimation (KDE)
vals = np.linspace(0,30,1000)
fkde = gaussian_kde(fwhm)
fpdf = fkde(vals)
mode = vals[np.argmax(fpdf)]
std = np.std(fwhm)
plt.ion()
plt.figure(99)
plt.clf()
plt.hist(fwhm, color='darkgoldenrod',bins=35)
plt.xlabel('FWHM (arcsec)',fontsize=16)
plt.ylabel('Frequency',fontsize=16)
plt.annotate('mode $=$ %.2f" ' % mode, [0.87,0.85],horizontalalignment='right',
xycoords='figure fraction',fontsize='large')
plt.annotate('median $=$ %.2f" ' % med, [0.87,0.8],horizontalalignment='right',
xycoords='figure fraction',fontsize='large')
plt.annotate('mean $=$ %.2f" ' % mean, [0.87,0.75],horizontalalignment='right',
xycoords='figure fraction',fontsize='large')
xvals = np.linspace(0,30,1000)
kde = gaussian_kde(fwhm)
pdf = kde(xvals)
dist_c = np.cumsum(pdf)/np.sum(pdf)
func = interp1d(dist_c,vals,kind='linear')
lo = np.float(func(math.erfc(1./np.sqrt(2))))
hi = np.float(func(math.erf(1./np.sqrt(2))))
disthi = np.linspace(.684,.999,100)
distlo = disthi-0.6827
disthis = func(disthi)
distlos = func(distlo)
interval = np.min(disthis-distlos)
plt.annotate('1 $\sigma$ int. $=$ %.2f" ' % interval, [0.87,0.70],horizontalalignment='right',
xycoords='figure fraction',fontsize='large')
plt.rcdefaults()
plt.savefig(outpath+'Seeing_Cumulative.png',dpi=300)
return
def show_sensfunc(self):
"""
Plot the sensitivity function
"""
if self.sens_dict is None:
msgs.warn("You need to generate the sensfunc first!")
return None
plt.rcdefaults()
plt.rcParams["xtick.top"] = True
plt.rcParams["ytick.right"] = True
plt.rcParams["xtick.minor.visible"] = True
plt.rcParams["ytick.minor.visible"] = True
plt.rcParams["ytick.direction"] = 'in'
plt.rcParams["xtick.direction"] = 'in'
plt.rcParams["xtick.labelsize"] = 13
plt.rcParams["ytick.labelsize"] = 13
plt.rcParams['font.family'] = 'times new roman'
norder = self.sens_dict['norder']
for iord in range(norder):
sens_dict_iord = self.sens_dict[str(iord)]
plt.plot(sens_dict_iord['wave'], sens_dict_iord['sensfunc'])
plt.xlabel('Wavelength [ang]', fontsize=14)
plt.ylabel('Sensfunc', fontsize=14)
plt.ylim([0., 100.0])
plt.show()
def plot_clusterings(target, source, env):
"""
Plot items with clustering from first file, using 2-d coordinates from second file.
The functions GET_COORDS and GET_CLUSTS specify operations to turn each file object
into a mapping from item name to coordinate tuple or cluster number, respectively.
"""
pyplot.rcdefaults()
pyplot.figure(figsize=(10, 10))
args = source[-1].read()
# rcdefaults()
clusts = dict(ri2py(eval("lambda x : %s" % args.get("GET_CLUSTERS", "x"))(numpy.load(source[0].rstr()))).rx2("cluster").iteritems())
coords = eval("lambda x : %s" % args.get("GET_COORDS", "x"))(numpy.load(source[1].rstr()))
labels = coords.keys()
#if args.get("NORMALIZE", False):
# for i in [0, 1]:
# ttcoords[:, i] = (ttcoords[:, i] - ttcoords[:, i].min()) / numpy.max(ttcoords[:, i] - ttcoords[:, i].min())
[pyplot.scatter(coords[l][0], coords[l][1], label=l, s=64, marker=shapes[clusts[l]], color=colors[clusts[l]]) for i, l in enumerate(labels)]
ymin, ymax = pyplot.ylim()
inc = (ymax - ymin) / 40.0
[pyplot.text(coords[l][0], coords[l][1] + inc, l, fontsize=12, ha="center") for i, l in enumerate(labels)]
pyplot.xticks([], [])
pyplot.yticks([], [])
pyplot.xlabel("First principal component")
pyplot.ylabel("Second principal component")
pyplot.savefig(target[0].rstr(), bbox_inches="tight")
pyplot.cla()
return None
def plot():
#Read in the airmass terms and the MJDs
data = ascii.read('airmass.dat')
mjds, airmasses, sites = data['mjd'], data['airmass'], data['site']
setup_plot()
colors = {'lsc':'blue', 'cpt':'red', 'coj': 'green'}
for site in ['lsc', 'cpt', 'coj']:
where_site = sites == site
pyplot.plot(mjds[where_site] - 57000, airmasses[where_site],
'o', color=colors[site])
pyplot.xlim(7.7, 10.3)
pyplot.ylim(2.35, 0.95)
pyplot.xlabel('MJD - 57000')
pyplot.ylabel('Airmass')
a = pyplot.annotate("", xy=(8.75, 1.2), xycoords='data',xytext=(8.30, 1.2), textcoords='data',
arrowprops={'arrowstyle':"<->"})
a.arrow_patch.set_linewidth(2)
pyplot.text(8.525, 1.17,'Bad Weather', ha='center', fontsize='medium')
pyplot.legend(labels=['Chile', 'South Africa', 'Australia'], loc=3)
pyplot.savefig('he0435_airmass.pdf', bbox_inches='tight', pad_inches=0.05)
pyplot.show()
pyplot.rcdefaults()
def mplSetupStandard():
mpl.rcdefaults()
mpl.rc('figure',
dpi = 100,
figsize = (6,6),
facecolor = 'white',
autolayout = True
)
def __init__(self):
threading.Thread.__init__(self)
self.setDaemon(True)
self._homeDir = os.path.expanduser("~/.sensomatic")
self._configFileName = self._homeDir + '/config.ini'
self._config = ConfigParser.ConfigParser()
self._readConfig()
plt.rcdefaults()
def process_blocks(blocks, src_path, image_path, cfg):
"""Run source, save plots as images, and convert blocks to rst.
Parameters
----------
blocks : list of block tuples
Code and text blocks from example. See `split_code_and_text_blocks`.
src_path : str
Path to example file.
image_path : str
Path where plots are saved (format string which accepts figure number).
cfg : config object
Sphinx config object created by Sphinx.
Returns
-------
figure_list : list
List of figure names saved by the example.
rst_text : str
Text with code wrapped code-block directives.
"""
src_dir, src_name = src_path.psplit()
if not src_name.startswith('plot'):
convert_func = dict(code=codestr2rst, text=docstr2rst)
rst_blocks = [convert_func[blabel](bcontent)
for i, (blabel, brange, bcontent) in enumerate(blocks)]
return [], '\n'.join(rst_blocks)
# index of blocks which have inline plots
inline_tag = cfg.plot2rst_plot_tag
idx_inline_plot = [i for i, b in enumerate(blocks)
if inline_tag in b[2]]
image_dir, image_fmt_str = image_path.psplit()
figure_list = []
plt.rcdefaults()
plt.rcParams.update(cfg.plot2rst_rcparams)
plt.close('all')
example_globals = {}
rst_blocks = []
fig_num = 1
for i, (blabel, brange, bcontent) in enumerate(blocks):
if blabel == 'code':
exec(bcontent, example_globals)
rst_blocks.append(codestr2rst(bcontent))
else:
if i in idx_inline_plot:
plt.savefig(image_path.format(fig_num))
figure_name = image_fmt_str.format(fig_num)
fig_num += 1
figure_list.append(figure_name)
figure_link = os.path.join('images', figure_name)
bcontent = bcontent.replace(inline_tag, figure_link)
rst_blocks.append(docstr2rst(bcontent))
return figure_list, '\n'.join(rst_blocks)
def clear(self):
"""Custom clear method that resets everything back o defaults."""
attrs = [x for x in dir(self) if self._allowed_attr(x)]
defaults = self.__class__()
for attr in attrs:
setattr(self, attr, getattr(defaults, attr))
plt.rcdefaults()
attrs = [x for x in dir(self) if self._allowed_attr(x, template=True)]
for attr in attrs:
delattr(self, attr)
def plot_derivs():
import numpy as np
import matplotlib.pyplot as plt
x = np.arange(0, 100, 0.1)
params = (x, 15, 50, 1)
gauss1 = gauss(*params)
gauss1d = gauss_1st_deriv(*params)
gauss2d = gauss_2nd_deriv(*params)
gauss3d = gauss_3rd_deriv(*params)
gauss4d = gauss_4th_deriv(*params)
deriv_list = [gauss1d, gauss2d, gauss3d, gauss4d,]
for i, comp in enumerate(deriv_list):
deriv_list[i] = comp / np.max(comp)
# Set up plot aesthetics
plt.clf()
plt.close()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
font_scale = 15
params = {#'backend': .pdf',
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
'text.fontsize': font_scale,
'legend.fontsize': font_scale * 4.0 / 4.0,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': False,
#'figure.figsize': (8, 8 * y_scaling),
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
fig, ax = plt.subplots(1, 1, figsize=(7, 7))
ax.plot(x, gauss1, linewidth=3, color='k')
ax.set_ylim(-2.5, 1.1)
ax.set_xlabel('Velocity (km/s)')
ax.set_ylabel('Intensity')
plt.savefig('gauss_deriv0.png')
for i, deriv in enumerate(deriv_list):
ax.plot(x, deriv, linewidth=1)
plt.savefig('gauss_deriv' + str(i+1) + '.png')
def plot_temps():
import numpy as np
import matplotlib.pyplot as plt
x = np.arange(0, 100, 0.1)
gauss1 = gauss(x, 5, 50, 1)
gauss2 = gauss(x, 10, 50, 1)
gauss3 = gauss(x, 15, 50, 1)
gauss1 /= np.sum(gauss1)
gauss2 /= np.sum(gauss2)
gauss3 /= np.sum(gauss3)
scale = np.max(np.array((gauss1, gauss2, gauss3)))
gauss1 /= scale
gauss2 /= scale
gauss3 /= scale
# Set up plot aesthetics
plt.clf()
plt.close()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
font_scale = 20
params = {#'backend': .pdf',
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
'text.fontsize': font_scale,
'legend.fontsize': font_scale * 4.0 / 4.0,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': False,
#'figure.figsize': (8, 8 * y_scaling),
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
fig, ax = plt.subplots(1, 1, figsize=(7, 7))
ax.plot(x, gauss1, label='Cold', linewidth=3)
ax.plot(x, gauss2, label='Warm', linewidth=3)
ax.plot(x, gauss3, label='Hot', linewidth=3)
ax.set_xlabel('Velocity (km/s)')
ax.set_ylabel('Intensity')
ax.legend()
plt.savefig('gauss_temps.png')
def setup_jointplot():
plt.rcdefaults()
np.random.seed(0)
N = 37
df = pandas.DataFrame({
'A': np.random.normal(size=N),
'B': np.random.lognormal(mean=0.25, sigma=1.25, size=N),
'C': np.random.lognormal(mean=1.25, sigma=0.75, size=N)
})
return df
def MakeSubplots(distances, plot_gender='male', special=False):
pyplot.rc('figure', figsize=(5, 10))
pyplot.rc('font', size=9.0)
pyplot.rc('xtick.major', size=0)
pyplot.rc('ytick.major', size=0)
pyplot.subplots_adjust(wspace=0.4, hspace=0.4,
right=0.95, left=0.15,
top=0.95, bottom=0.05)
t = miles.items()
t.sort(key=lambda x: x[1])
titles = [x[0] for x in t]
gender = plot_gender
i=0
for distance in titles:
i += 1
data = distances[distance, gender]
if gender != plot_gender:
continue
pyplot.subplot(6, 2, i)
if i%2 == 1:
pyplot.ylabel('mph')
xs, ys = zip(*data)
# extend the current record to the present
first_x = xs[1]
last_x = xs[-1]
if special:
pyplot.xticks([1950, 1970, 1990, 2011])
elif i==2:
pyplot.xticks([int(first_x), 2011])
else:
pyplot.xticks([int(first_x), 1960, 2011])
first_y = ys[0]
last_y = ys[-1]
pyplot.plot([last_x, 2012.4], [last_y, last_y], 'b-')
if special:
pyplot.plot([1950, first_x], [first_y, first_y], 'b-')
pyplot.plot(xs, ys, 'o-', markersize=4)
pyplot.title(distance)
root = 'world_record_speed'
myplot.Save(root=root)
pyplot.rcdefaults()
def plot_nstars(cat_ra, cat_mjd, suff):
nstars = [max([len(frame) for frame in ra]) for ra in cat_ra]
mjd = [np.average(mjd) for mjd in cat_mjd]
plt.rcdefaults()
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(mjd, nstars, '.')
ax.set_xlabel('MJD')
ax.set_ylabel('Maximum number of stars per night')
fig.savefig(param['output_path']+'maximum_number_of_stars_per_night'+suff, bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
def plot(self, filename):
# Check that the optical properties have been set
self.optical_properties.ensure_all_set()
import matplotlib.pyplot as plt
# Save original rc parameters
rc_orig = plt.rcParams
# Reset to defaults
plt.rcdefaults()
plt.rc('legend', fontsize=7)
plt.rc('axes', titlesize='x-small')
plt.rc('axes', labelsize='x-small')
plt.rc('xtick', labelsize='xx-small')
plt.rc('ytick', labelsize='xx-small')
plt.rc('axes', linewidth=0.5)
plt.rc('patch', linewidth=0.5)
# Check that emissivities are set (before computing mean opacities)
if not self.emissivities.all_set():
logger.info("Computing emissivities assuming LTE")
self.emissivities.set_lte(self.optical_properties)
# Compute mean opacities if not already existent
if not self.mean_opacities.all_set():
logger.info("Computing mean opacities")
self.mean_opacities.compute(self.emissivities, self.optical_properties)
# Initialize figure
fig = plt.figure(figsize=(10, 12))
# Plot optical properties
fig = self.optical_properties.plot(fig, [421, 423, 424, 425, 426])
# Plot emissivities
fig = self.emissivities.plot(fig, 427)
# Plot mean opacities
fig = self.mean_opacities.plot(fig, 428)
# Adjust spacing between subplots
fig.subplots_adjust(left=0.08, right=0.92, wspace=0.22, hspace=0.30)
# Save figure
fig.savefig(filename, bbox_inches='tight')
# Close figure to save RAM
plt.close(fig)
# Restore rc parameters
plt.rc(rc_orig)
def __delitem__(self, name):
if hasattr(self, name):
default = getattr(self.__class__(), name)
setattr(self, name, default)
elif name in plt.rcParams:
params = dict(plt.rcParams)
del params[name]
plt.rcdefaults()
plt.rcParams.update(params)
super(DefaultPlotStyle, self).__delattr__(_remove_dots("template_{}".format(name)))
else:
raise KeyError("{} is not recognised as part of the template".format(name))
def plotLonLat(self, lonData, latData, indicator):
"""
Plot the input lat/lon values lagged against themselves,
and the same for the changes in lat/lon.
"""
pyplot.figure(self.figurenum(), figsize=(7, 12))
dlon = lonData[1:] - lonData[:-1]
dlat = latData[1:] - latData[:-1]
j = numpy.where(indicator[1:] == 0)
dlon = dlon[j]
dlat = dlat[j]
# Correct change in longitude where the value jumps across the 180E
# meridian
k = numpy.where(dlon<-180.)
dlon[k] += 360.
pyplot.subplot(211)
pyplot.plot(dlon[1:], dlon[:-1], 'k.', markersize=1)
m, c, r, p, e = _linreg(dlon)
pyplot.text(-3, 3, "r = %5.3f"%r, ha='center',
va='center', color='r', size=14)
pyplot.xlim(-4., 4.)
pyplot.ylim(-4., 4.)
pyplot.xticks(numpy.arange(-4., 4.1, 1.))
pyplot.yticks(numpy.arange(-4., 4.1, 1.))
pyplot.ylabel(r"$\Delta lon (t)$", fontsize=16)
pyplot.xlabel(r"$\Delta lon (t-1)$", fontsize=16)
#pyplot.grid(True)
pyplot.title("Longitude rate of change")
pyplot.subplot(212)
pyplot.plot(dlat[1:], dlat[:-1], 'k.', markersize=1)
m, c, r, p, e = _linreg(dlat)
pyplot.text(-3, 3, "r = %5.3f"%r, ha='center',
va='center', color='r', size=14)
pyplot.xlim(-4., 4.)
pyplot.ylim(-4., 4.)
pyplot.xticks(numpy.arange(-4., 4.1, 1.))
pyplot.yticks(numpy.arange(-4., 4.1, 1.))
pyplot.ylabel(r"$\Delta lat (t)$", fontsize=16)
pyplot.xlabel(r"$\Delta lat (t-1)$", fontsize=16)
pyplot.title("Latitude rate of change")
self.savefig('lonlat_corr')
pyplot.rcdefaults()
self.scatterHistogram(dlon[1:], dlon[:-1], 'dlon_scatterHist')
self.scatterHistogram(dlat[1:], dlat[:-1], 'dlat_scatterHist')
def new_figure(self, figure=False, **kargs):
"""This is called by PlotMixin to setup a new figure before we do anything."""
plt.rcdefaults() # Start by resetting to our default settings
params = dict()
self.apply()
if "fig_width_pt" in dir(self):
self.fig_width = self.fig_width_pt * self._inches_per_pt
if "fig_height_pt" in dir(self):
self.fig_height = self.fig_width * self._golden_mean # height in inches
if "fig_ratio" in dir(self) and "fig_width" in dir(self):
self.fig_height = self.fig_width / self.fig_ratio
if "fig_width" and "fig_height" in self.__dict__:
self.template_figure__figsize = (self.fig_width, self.fig_height)
for attr in dir(self):
if attr.startswith("template_"):
attrname = _add_dots(attr[9:])
value = self.__getattribute__(attr)
if attrname in plt.rcParams.keys():
params[attrname] = value
plt.rcParams.update(params) # Apply these parameters
projection = kargs.pop("projection", "rectilinear")
self.template_figure__figsize = kargs.pop("figsize", self.template_figure__figsize)
if "ax" in kargs: # Giving an axis instance in kargs means we can use that as out figure
ax = kargs.get("ax")
plt.sca(ax)
figure = plt.gcf().number
if isinstance(figure, bool) and not figure:
ret = None
elif figure is not None:
fig = plt.figure(figure, figsize=self.template_figure__figsize)
if len(fig.axes) == 0:
rect = [plt.rcParams["figure.subplot.{}".format(i)] for i in ["left", "bottom", "right", "top"]]
rect[2] = rect[2] - rect[0]
rect[3] = rect[3] - rect[1]
if projection == "3d":
ax = fig.add_subplot(111, projection="3d")
else:
ax = fig.add_axes(rect)
else:
if projection == "3d":
ax = kargs.pop("ax", fig.gca(projection="3d"))
else:
ax = kargs.pop("ax", fig.gca())
ret = fig
else:
if projection == "3d":
ret = plt.figure(figsize=self.template_figure__figsize, **kargs)
ax = ret.add_subplot(111, projection="3d")
else:
ret, ax = plt.subplots(figsize=self.template_figure__figsize, **kargs)
return ret, ax
def test_graph(config, filename, n):
plt.rcdefaults()
people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim')
y_pos = np.arange(len(people))
performance = 3 + 10 * np.random.rand(len(people))
error = np.random.rand(len(people))
plt.barh(y_pos, performance, xerr=error, align='center', alpha=0.4)
plt.yticks(y_pos, people)
plt.xlabel("Step "+str(n))
plt.title('Are samples measurements working?')
plt.savefig(filename)
def problem_2c():
import matplotlib.pyplot as plt
m = 1000
a = np.random.uniform(0, 1, size=m)
A = np.matrix([np.ones((m)), a]).T
b = a ** 2 + np.random.normal(0, 1, size=m)
B = np.matrix(b).T
# Fit x
x = np.linalg.inv(A.T * A) * A.T * B
# create b_fit
a_fit = np.arange(0, 1, 0.01)
b_fit = x[0, 0] + x[1, 0] * a_fit ** 2
# Plot!
# Set up plot aesthetics
plt.clf()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
# color_cycle = [colormap(i) for i in np.linspace(0, 0.9, len(flux_list))]
font_scale = 10
params = { #'backend': .pdf',
"axes.labelsize": font_scale,
"axes.titlesize": font_scale,
"text.fontsize": font_scale,
"legend.fontsize": font_scale * 3 / 4.0,
"xtick.labelsize": font_scale,
"ytick.labelsize": font_scale,
"font.weight": 500,
"axes.labelweight": 500,
"text.usetex": False,
#'figure.figsize': (8, 8 * y_scaling),
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
fig = plt.figure(figsize=(3, 2))
ax = fig.add_subplot(111)
ax.plot(a, b, linestyle="", marker="^", alpha=0.4)
ax.plot(a_fit, b_fit, color="r")
ax.set_xlabel(r"$a_i$")
ax.set_ylabel(r"$b_i$")
plt.savefig("problem2c_fig.png", bbox_inches="tight")
def plot(self):
try:
import matplotlib.pyplot as plt
except ImportError:
return
reactants_label = ' + '.join([reactant.label for reactant in self.job.reaction.reactants])
ts_label = self.job.reaction.transitionState.label
products_label = ' + '.join([reactant.label for reactant in self.job.reaction.products])
plt.rcdefaults()
_, ax = plt.subplots(nrows=len(self.conditions), ncols=2, tight_layout=True)
labels = [reactants_label, ts_label, products_label]
min_sa = min(min(min(self.f_sa_coefficients.itervalues())), min(min(self.r_sa_coefficients.itervalues())))
max_sa = max(max(max(self.f_sa_coefficients.itervalues())), max(max(self.r_sa_coefficients.itervalues())))
for i, condition in enumerate(self.conditions):
f_values = [self.f_sa_coefficients[self.job.reaction.reactants[0]][i],
self.f_sa_coefficients[self.job.reaction.transitionState][i],
self.f_sa_coefficients[self.job.reaction.products[0]][i]]
r_values = [self.r_sa_coefficients[self.job.reaction.reactants[0]][i],
self.r_sa_coefficients[self.job.reaction.transitionState][i],
self.r_sa_coefficients[self.job.reaction.products[0]][i]]
y_pos = np.arange(3)
ax[i][0].barh(y_pos, f_values, align='center', color='green')
ax[i][0].set_yticks(y_pos)
ax[i][0].set_yticklabels(labels)
ax[i][0].invert_yaxis() # labels read top-to-bottom
ax[i][0].set_xlabel(r'Sensitivity: $\frac{\partial\:\ln{k}}{\partial\:E0}$, ($\frac{J}{mol}$)')
ax[i][0].set_title('Forward, {0}'.format(condition))
ax[i][0].set_xlim([min_sa, max_sa])
ax[i][1].barh(y_pos, r_values, align='center', color='blue')
ax[i][1].set_yticks(y_pos)
ax[i][1].set_yticklabels(labels)
ax[i][1].invert_yaxis() # labels read top-to-bottom
ax[i][1].set_xlabel(r'Sensitivity: $\frac{\partial\:\ln{k}}{\partial\:E0}$, ($\frac{J}{mol}$)')
ax[i][1].set_title('Reverse, {0}'.format(condition))
ax[i][1].set_xlim([min_sa, max_sa])
plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
if not os.path.exists(self.sensitivity_path):
os.mkdir(self.sensitivity_path)
valid_chars = "-_.()<=> %s%s" % (string.ascii_letters, string.digits)
reaction_str = '{0} {1} {2}'.format(
' + '.join([reactant.label for reactant in self.job.reaction.reactants]),
'<=>', ' + '.join([product.label for product in self.job.reaction.products]))
filename = ''.join(c for c in reaction_str if c in valid_chars) + '.pdf'
path = os.path.join(self.sensitivity_path, filename)
plt.savefig(path)
plt.close()
def apply_standard_plotstyle(ax, nonpercents=0, default=0):
if default == 0:
standard_axes = [1, nSessions, 0, 1.0]
ax.set_xlabel("sessions")
plt.rc("legend", fontsize=8, loc="upper right", fancybox=True)
plt.rc("axes", titlesize="medium", labelsize="small")
plt.rc("xtick", labelsize="small")
plt.rc("ytick", labelsize="small")
plt.rc("lines", markersize=3, markeredgewidth=0.3)
plt.axis([1, nSessions, 0, 1.0])
if nonpercents == 1:
plt.axis([1, nSessions, 0, maxy])
elif default == 1:
plt.xlabel("")
plt.rcdefaults()
def apply_standard_plotstyle(ax, nonpercents=0, default=0):
if default == 0:
standard_axes = [1, nSessions, 0, 1.0]
ax.set_xlabel('sessions')
plt.rc('legend', fontsize=8, loc='upper right', fancybox=True)
plt.rc('axes', titlesize='medium', labelsize='small')
plt.rc('xtick', labelsize='small')
plt.rc('ytick', labelsize='small')
plt.rc('lines', markersize=3, markeredgewidth=0.3)
plt.axis([1, nSessions, 0, 1.0])
if nonpercents==1:
plt.axis([1, nSessions, 0, maxy])
elif default == 1:
plt.xlabel('')
plt.rcdefaults()
def clean_modules():
"""Remove "unload" seaborn from the name space
After a script is executed it can load a variety of setting that one
does not want to influence in other examples in the gallery."""
# Horrible code to 'unload' seaborn, so that it resets
# its default when is load
# Python does not support unloading of modules
# https://bugs.python.org/issue9072
for module in list(sys.modules.keys()):
if 'seaborn' in module:
del sys.modules[module]
# Reset Matplotlib to default
plt.rcdefaults()
def plot_crime_type(data):
# extract data
crime_types = {}
for rec in data:
c_type = rec["Category"]
crime_types[c_type] = crime_types.get(c_type, 0) + 1
# plot data
plt.rcdefaults()
categories = crime_types.keys()
y_pos = range(len(categories))
plt.barh(y_pos, [crime_types[cat] for cat in crime_types],height=2, align='center', alpha=0.4)
plt.yticks(y_pos, categories)
plt.xlabel('Number of occurrence')
plt.title('Total number of occurrence of every crime')
plt.show()
def start(self):
plt.rcdefaults()
# Example data
people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim')
y_pos = np.arange(len(people))
performance = 3 + 10 * np.random.rand(len(people))
error = np.random.rand(len(people))
plt.bar(y_pos, performance, align='center', alpha=0.4)
plt.yticks(y_pos, people)
plt.xlabel('Performance')
plt.title('How fast do you want to go today?')
plt.show()
def horizontal_bar():
"""
Simple demo of a horizontal bar chart.
"""
plt.rcdefaults()
# Example data
people = ('Tom', 'Dick', 'Harry', 'Slim', 'Jim')
y_pos = np.arange(len(people))
performance = 3 + 10 * np.random.rand(len(people))
error = np.random.rand(len(people))
plt.barh(y_pos, performance, xerr=error, align='center')
plt.yticks(y_pos, people)
plt.xlabel('Performance')
plt.title('How fast do you want to go today?')
plt.show()
def plot(dataset, split, path, out_path):
if dataset in [
'bace_c', 'bbbp', 'clintox', 'hiv', 'muv', 'pcba', 'pcba_146',
'pcba_2475', 'sider', 'tox21', 'toxcast'
]:
mode = 'classification'
else:
mode = 'regression'
data = {}
with open(path, 'r') as f:
reader = csv.reader(f)
for line in reader:
if line[0] == dataset and line[1] == split:
data[line[3]] = line[8]
labels = []
values = []
colors = []
for model in ORDER:
if model in data.keys():
labels.append(model)
colors.append(COLOR[model])
values.append(float(data[model]))
y_pos = np.arange(len(labels))
plt.rcdefaults()
fig, ax = plt.subplots()
ax.barh(y_pos, values, align='center', color='green')
ax.set_yticks(y_pos)
ax.set_yticklabels(labels)
ax.invert_yaxis()
if mode == 'regression':
ax.set_xlabel('R square')
ax.set_xlim(left=0., right=1.)
else:
ax.set_xlabel('ROC-AUC')
ax.set_xlim(left=0.4, right=1.)
t = time.localtime(time.time())
ax.set_title("Performance on %s (%s split), %i-%i-%i" %
(dataset, split, t.tm_year, t.tm_mon, t.tm_mday))
plt.tight_layout()
for i in range(len(colors)):
ax.get_children()[i].set_color(colors[i])
ax.text(
values[i] - 0.1, y_pos[i] + 0.1, str("%.3f" % values[i]), color='white')
fig.savefig(os.path.join(out_path, dataset + '_' + split + '.png'))
def plot_crime_district(data):
# extract data
crime_count = {}
for rec in data:
d = rec["PdDistrict"]
crime_count[d] = crime_count.get(d, 0) + 1
districts = crime_count.keys()
# plot data
plt.rcdefaults()
y_pos = map(lambda a: a, range(len(districts)))
plt.barh(y_pos, [crime_count[d] for d in districts], align='center', alpha=0.4)
plt.yticks(y_pos, districts, fontsize=8)
plt.xlabel('Number of occurrence', fontsize=10)
plt.title('Total number of occurrence in every district', fontsize=12)
plt.show()
def latex_off():
plt.rcdefaults()
plt.rcParams.update(matplotlib.rcParamsDefault)
def aufgabe1():
#
# In der ersten Aufgabe sollen Sie sich mit dem Brown Corpus
# vertraut machen.
# - Laden Sie den Corpus und schauen Sie sich dessen Aufbau an.
# - Analysieren Sie den Corpus in dem Sie Wortstatistiken bestimmen.
# - Verbessern Sie die Aussagekraft der Statistiken.
# Laden des Corpus
# Fuer das Fachprojekt benoetigen Sie die NLTK (http://www.nltk.org/)
# Datensaetze "brown" und "stopwords". Falls diese noch nicht lokal
# auf Ihrem Rechner verfuegbar sein sollten, koennen Sie sie ueber
# den "NLTK Downloader" herunterladen. Ein entsprechender Dialog
# oeffnet sich in diesem Fall automatisch.
CorpusLoader.load()
#
# Im Folgenden werden einige grundlegende Statistiken des Brown Corpus
# ausgegeben, die vor allem etwas ueber dessen Struktur / Aufbau
# aussagen.
# Siehe auch: http://en.wikipedia.org/wiki/Brown_Corpus
#
# Der Corpus enthaelt verschiedene Kategorien, in die Dokumente
# einsortiert sind. Ein Dokument besteht aus Woertern.
# Als naechstes sehen Sie, wie Sie auf Kategorien, Dokumente und
# Woerter zugreifen koennen.
brown = CorpusLoader.brown_corpus()
brown_categories = brown.categories()
brown_documents = brown.fileids()
brown_words = brown.words()
# Geben Sie nun die Gesamtanzahl von Kategorien, Dokumenten und Woertern
# mit print auf der Konsole aus.
#print(type(brown_categories),type(brown_documents),type(brown_words))
print('Anzahlen:\nKategorien {}\nDokumente {}\nWoerter {}'.format(len(brown_categories),
len(brown_documents),
len(brown_words)))
# Geben Sie die Namen der einzelnen Kategorien aus.
print(brown_categories)
# Bisher haben Sie noch keine Information ueber die Struktur des Brown
# Corpus gewonnen, da sie jeweils die Gesamtzahl von Kategorien, Dokumenten
# und Woertern ausgegeben haben.
#
# Geben Sie als naechstes die Anzahl von Dokumenten und Woertern je
# Kategorie aus.
# http://www.nltk.org/howto/corpus.html#categorized-corpora
# Hilfreiche Funktionen: fileids, words
#
# Visualisieren Sie die Verteilungen mit Hilfe von horizontalen bar plots.
# http://matplotlib.org/examples/lines_bars_and_markers/barh_demo.html
#
# Optional: Plotten Sie die Verteilungen mit vertikalen bar plots.
# Vermeiden Sie, dass sich die an der x-Achse aufgetragenen labels ueberlappen
# http://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes.set_xticklabels
# Stellen Sie nun die Verteilungen ueber Dokumente und Woerter in einem
# gemeinsamen Plot dar. Verwenden Sie unterschiedliche Farben.
# http://matplotlib.org/examples/api/barchart_demo.html
cat_to_count = {cat : (len(brown.fileids(categories = cat)),
len(brown.words(categories = cat)))
for cat in brown_categories}
print('Kategorien und ihre Werte (Documente, Woerter)\n {}'.format(cat_to_count))
y_pos = np.arange(len(brown_categories))
cats = np.array(cat_to_count.keys())
files = np.array(cat_to_count.values())[:,0]
words = np.array(cat_to_count.values())[:,1]
# plotting bars for documents
plt.rcdefaults()
fig, ax = plt.subplots()
ax.set_yticks(y_pos)
ax.set_yticklabels(cats)
ax.invert_yaxis()
ax.barh(y_pos, files,
align='center', color='green')
plt.show()
# plotting bars for words
plt.rcdefaults()
fig, ax = plt.subplots()
ax.set_yticks(y_pos)
ax.set_yticklabels(cats)
ax.invert_yaxis()
ax.barh(y_pos, words,
align='center', color='green')
plt.show()
# ********************************** ACHTUNG **************************************
# Die nun zu implementierenden Funktionen spielen eine zentrale Rolle im weiteren
# Verlauf des Fachprojekts. Achten Sie auf eine effiziente und 'saubere' Umsetzung.
# Verwenden Sie geeignete Datenstrukturen und passende Python Funktionen.
# Wenn Ihnen Ihr Ansatz sehr aufwaendig vorkommt, haben Sie vermutlich nicht die
# passenden Datenstrukturen / Algorithmen / (highlevel) Python / NumPy Funktionen
# verwendet. Fragen Sie in diesem Fall!
#
# Schauen Sie sich jetzt schon gruendlich die Klassen und deren Interfaces in den
# mitgelieferten Modulen an. Wenn Sie Ihre Datenstrukturen von Anfang an dazu
# passend waehlen, erleichtert dies deren spaetere Benutzung. Zusaetzlich bieten
# diese Klassen bereits etwas Inspiration fuer Python-typisches Design, wie zum
# Beispiel Duck-Typing.
#
# Zu einigen der vorgebenen Intefaces finden Sie Unit Tests in dem Paket 'test'.
# Diese sind sehr hilfreich um zu ueberpruefen, ob ihre Implementierung zusammen
# mit anderen mitgelieferten Implementierungen / Interfaces funktionieren wird.
# Stellen Sie immer sicher, dass die Unit tests fuer die von Ihnen verwendeten
# Funktionen erfolgreich sind.
# Hinweis: Im Verlauf des Fachprojekts werden die Unit Tests nach und nach erfolg-
# reich sein. Falls es sie zu Beginn stoert, wenn einzelne Unit Tests fehlschlagen
# koennen Sie diese durch einen 'decorator' vor der Methodendefinition voruebergehend
# abschalten: @unittest.skip('')
# https://docs.python.org/2/library/unittest.html#skipping-tests-and-expected-failures
# Denken Sie aber daran sie spaeter wieder zu aktivieren.
#
# Wenn etwas unklar ist, fragen Sie!
# *********************************************************************************
# Um Texte / Dokumente semantisch zu analysieren, betrachtet man Verteilungen
# ueber Wortvorkommen. Ziel dieser semantischen Analyse soll es letztlich sein
# unbekannte Dokumente automatisch einer bekannten Kategorie / Klasse zuzuordnen.
#
# Bestimmen Sie die 20 haeufigsten Woerter des Brown Corpus (insgesamt), sowie
# die 20 haeufigsten Woerter je Kategorie.
# http://docs.python.org/2/library/collections.html#collections.defaultdict
# http://docs.python.org/2/library/functions.html#sorted
# Hinweis: Die Dokumentation zu defaultdict enthaelt ein sehr hilfreiches Beispiel.
#
# Implementieren Sie die (statische) Funktion BagOfWords.most_freq_words im Modul
# features.
w_most_20 = BagOfWords.most_freq_words(brown_words, n_words=20)
cat_to_w_most_20 = {cat : BagOfWords.most_freq_words(brown.words(categories=cat), n_words=20)
for cat in brown_categories}
print(w_most_20)
print(cat_to_w_most_20)
#
# Diese Woerter sind nicht besonders charakteristisch fuer die Unterscheidung
# verschiedener Kategorien. Daher entfernt man solche wenig aussagekraeftigen
# Woerter vor einer semantischen Analyse. Man bezeichnet diese Woerter als
# stopwords.
# Eine Liste mit stopwords wird durch NLTK bereitgestellt (siehe oben sowie
# im 'corpus' Modul).
# Filtern Sie nun alle stopwords bevor Sie die 20 haeufigsten Woerter im Brown
# Corpus (insgesamt und je Kategorie) erneut bestimmen. Achten Sie dabei auch
# Gross- und Kleinschreibung und filtern Sie ach Satzzeichen (string.punctuation).
# http://www.nltk.org/howto/corpus.html#word-lists-and-lexicons
# http://docs.python.org/2/library/string.html
#
# Geben Sie zunaechst stopwords und Satzzeichen auf der Kommandozeile aus.
# print(stopwords.words())
# print(string.punctuation)
# Mit der Liste von stopwords koennen Sie noch keine grammatikalischen Varianten
# von Woertern erfassen, die ebenfalls nicht entscheidend fuer die semantische
# Analyse von Texten sind (zum Beispiel: walking, walked).
#
# Verwenden Sie daher den PorterStemmer um Woerter auf ihre Wortstaemme abzubilden.
# Geben Sie die 20 haeufigsten Woerter nach jedem Filter Schrift aus:
# 1. stopwords und Satzzeichen
# filtered_words = (w for w in brown_words
# if w not in stopwords.words() and w not in string.punctuation)
# #print(BagOfWords.most_freq_words(filtered_words, n_words=20))
# # 2. Abbildung auf Wortstaemme (stemming)
# stemmer = PorterStemmer()
# stemmed_words = (stemmer.stem(w) for w in brown_words)
# print(BagOfWords.most_freq_words(stemmed_words, n_words=20))
#
normalizer = WordListNormalizer()
for c in brown_categories:
w_cat = normalizer.normalize_words(brown.words(categories = c))
print(c)
print(BagOfWords.most_freq_words(w_cat[0],20))
print(BagOfWords.most_freq_words(w_cat[1],20))
# Erlaeutern Sie Ihre Beobachtungen.
# http://www.nltk.org/api/nltk.stem.html#module-nltk.stem.porter
#
# Implementieren Sie die Funktion WordListNormalizer.normalize_words im
# features Modul.
return
def create_plot():
plt.rcdefaults()
fig, ax = plt.subplots()
fig.set_size_inches(18, 10)
return fig, ax
def reset_style():
plt.rcdefaults()
def main(argv=None):
r"""Routine for post-processing COCO data from two algorithms.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for comparing two algorithms (``*cmp.tex`` or ``*2*.tex``).
The used template file needs to be edited so that the command
``\bbobdatapath`` points to the output folder created by this routine.
The output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* must list folders containing BBOB data files. Each of these
folders should correspond to the data of ONE algorithm.
Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out operations.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`
--noise-free, --noisy
processes only part of the data.
--settings=SETTING
changes the style of the output figures and tables. At the
moment only the only differences are in the colors of the
output figures. SETTING can be either "grayscale", "color"
or "black-white". The default setting is "color".
--fig-only, --rld-only, --tab-only, --sca-only
these options can be used to output respectively the aRT
graphs figures, run length distribution figures or the
comparison tables scatter plot figures only. Any combination
of these options results in no output.
--conv
if this option is chosen, additionally convergence
plots for each function and algorithm are generated.
--no-rld-single-fcts
do not generate runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets.
--no-svg
do not generate the svg figures which are used in html files
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric2.py interface from the command line::
$ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest
will post-process the data from folders :file:`Alg0-baseline` and
:file:`Alg1-of-interest`, the former containing data for the
reference algorithm (zero-th) and the latter data for the
algorithm of concern (first). The results will be output in the
default output folder. The ``-v`` option adds verbosity.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())
This will execute the post-processing on the data found in folder
:file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
folder from the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist,
genericsettings.longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not args:
usage()
sys.exit()
# Process options
outputdir = genericsettings.outputdir
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--fig-only":
genericsettings.isRLDistr = False
genericsettings.isTab = False
genericsettings.isScatter = False
elif o == "--rld-only":
genericsettings.isFig = False
genericsettings.isTab = False
genericsettings.isScatter = False
elif o == "--tab-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isScatter = False
elif o == "--sca-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isTab = False
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--conv":
genericsettings.isConv = True
elif o == "--no-rld-single-fcts":
genericsettings.isRldOnSingleFcts = False
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--no-svg":
genericsettings.generate_svg_files = False
elif o == "--los-only":
warnings.warn("option --los-only will have no effect with rungeneric2.py")
elif o == "--crafting-effort=":
warnings.warn("option --crafting-effort will have no effect with rungeneric2.py")
elif o in ("-p", "--pickle"):
warnings.warn("option --pickle will have no effect with rungeneric2.py")
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if genericsettings.inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
config.config()
elif genericsettings.inputsettings == "grayscale": # probably very much obsolete
from bbob_pproc import grayscalesettings as inset # input settings
elif genericsettings.inputsettings == "black-white": # probably very much obsolete
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % genericsettings.inputsettings
+ 'argument for input flag "--settings".')
raise Usage(txt)
if not genericsettings.verbose:
warnings.simplefilter('module')
warnings.simplefilter('ignore')
print("\nPost-processing will generate comparison " +
"data in folder %s" % outputdir)
print(" this might take several minutes.")
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=genericsettings.verbose)
if 1 < 3 and len(sortedAlgs) != 2:
raise ValueError('rungeneric2.py needs exactly two algorithms to '
+ 'compare, found: ' + str(sortedAlgs)
+ '\n use rungeneric.py (or rungenericmany.py) to '
+ 'compare more algorithms. ')
if not dsList:
sys.exit()
if (any(ds.isBiobjective() for ds in dsList)
and any(not ds.isBiobjective() for ds in dsList)):
sys.exit()
for i in dictAlg:
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())
for i in dsList:
if i.dim not in genericsettings.dimensions_to_display:
continue
# check whether current set of instances correspond to correct
# setting of a BBOB workshop and issue a warning otherwise:
curr_instances = (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)))
correct = False
for instance_set_of_interest in inset.instancesOfInterest:
if curr_instances == instance_set_of_interest:
correct = True
if not correct:
warnings.warn('The data of %s do not list ' % i +
'the correct instances ' +
'of function F%d.' % i.funcId)
if len(sortedAlgs) < 2:
raise Usage('Expect data from two different algorithms, could ' +
'only find one.')
elif len(sortedAlgs) > 2:
warnings.warn('Data from folders: %s ' % (sortedAlgs) +
'were found, the first two will be processed.')
# Group by algorithm
dsList0 = dictAlg[sortedAlgs[0]]
if not dsList0:
raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))
dsList1 = dictAlg[sortedAlgs[1]]
if not dsList1:
raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))
# get the name of each algorithm from the input arguments
tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))
for i in dsList0:
i.algId = alg0name
for i in dsList1:
i.algId = alg1name
config.target_values(genericsettings.isExpensive)
config.config(dsList[0].testbed_name())
######################### Post-processing #############################
if (genericsettings.isFig or genericsettings.isRLDistr
or genericsettings.isTab or genericsettings.isScatter):
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if genericsettings.verbose:
print('Folder %s was created.' % (outputdir))
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname1(alg)) + '}')
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
lines, 1000, 'bbob_proc_commands.tex truncated, '
+ 'consider removing the file before the text run'
)
# Check whether both input arguments list noisy and noise-free data
dictFN0 = dsList0.dictByNoise()
dictFN1 = dsList1.dictByNoise()
k0 = set(dictFN0.keys())
k1 = set(dictFN1.keys())
symdiff = k1 ^ k0 # symmetric difference
if symdiff:
tmpdict = {}
for i, noisegrp in enumerate(symdiff):
if noisegrp == 'nzall':
tmp = 'noisy'
elif noisegrp == 'noiselessall':
tmp = 'noiseless'
if dictFN0.has_key(noisegrp):
tmp2 = sortedAlgs[0]
elif dictFN1.has_key(noisegrp):
tmp2 = sortedAlgs[1]
tmpdict.setdefault(tmp2, []).append(tmp)
txt = []
for i, j in tmpdict.iteritems():
txt.append('Only input folder %s lists %s data.'
% (i, ' and '.join(j)))
raise Usage('Data Mismatch: \n ' + ' '.join(txt) + '\nTry using --noise-free or --noisy flags.')
algName0 = toolsdivers.str_to_latex(
set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, ''))
algName1 = toolsdivers.str_to_latex(
set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, ''))
algorithm_name = "%s vs %s" % (algName1, algName0)
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.ppfigs_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPFIGS,
isBiobjective=dsList0.isBiobjective(),
functionGroups=dsList0.getFuncGroups(),
parentFileName=genericsettings.two_algorithm_file_name
)
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.ppscatter_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPSCATTER,
isBiobjective=dsList0.isBiobjective(),
functionGroups=dsList0.getFuncGroups(),
parentFileName=genericsettings.two_algorithm_file_name
)
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.pprldistr2_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPRLDISTR2,
isBiobjective=dsList0.isBiobjective(),
functionGroups=dsList0.getFuncGroups(),
parentFileName=genericsettings.two_algorithm_file_name
)
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.pptable2_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPTABLE2,
isBiobjective=dsList0.isBiobjective(),
functionGroups=dsList0.getFuncGroups(),
parentFileName=genericsettings.two_algorithm_file_name
)
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.pptables_file_name),
'', # algorithms names are clearly visible in the figure
htmlPage=ppfig.HtmlPage.PPTABLES,
isBiobjective=dsList[0].isBiobjective(),
functionGroups=dsList0.getFuncGroups(),
parentFileName=genericsettings.many_algorithm_file_name
)
if genericsettings.isFig:
print("log aRT1/aRT0 vs target function values...")
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
plt.rc('pdf', fonttype=42)
ppfig2.main(dsList0, dsList1, testbedsettings.current_testbed.ppfig2_ftarget,
outputdir, genericsettings.verbose)
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc('pdf', fonttype=42)
if genericsettings.isRLDistr:
print("ECDF runlength ratio graphs...")
if len(dictFN0) > 1 or len(dictFN1) > 1:
warnings.warn('Data for functions from both the noisy and ' +
'non-noisy testbeds have been found. Their ' +
'results will be mixed in the "all functions" ' +
'ECDF figures.')
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
# ECDFs of aRT ratios
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
if dim in inset.rldDimsOfInterest:
# ECDF for all functions altogether
try:
pprldistr2.main(dictDim0[dim], dictDim1[dim], dim,
testbedsettings.current_testbed.rldValsOfInterest,
outputdir,
'%02dD_all' % dim,
genericsettings.verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.' % dim)
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim,
testbedsettings.current_testbed.rldValsOfInterest,
outputdir,
'%02dD_%s' % (dim, fGroup),
genericsettings.verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim,
testbedsettings.current_testbed.rldValsOfInterest,
outputdir,
'%02dD_%s' % (dim, fGroup),
genericsettings.verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
pprldistr.caption_two(), # depends on the config
# setting, should depend
# on maxfevals
'}'
])
print_done()
# ECDFs per noise groups, code copied from rungenericmany.py
# (needed for bbob-biobj multiple algo template)
print("ECDF graphs per noise group...")
rungenericmany.grouped_ecdf_graphs(
pproc.dictAlgByNoi(dictAlg),
dsList[0].isBiobjective(),
sortedAlgs,
outputdir,
dictAlg[sortedAlgs[0]].getFuncGroups())
print_done()
# ECDFs per function groups, code copied from rungenericmany.py
# (needed for bbob-biobj multiple algo template)
print("ECDF runlength graphs per function group...")
rungenericmany.grouped_ecdf_graphs(
pproc.dictAlgByFuncGroup(dictAlg),
dsList[0].isBiobjective(),
sortedAlgs,
outputdir,
dictAlg[sortedAlgs[0]].getFuncGroups())
print_done()
print("ECDF runlength graphs...")
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
# ECDFs of all functions altogether
if dim in inset.rldDimsOfInterest:
try:
pprldistr.comp(dictDim1[dim], dictDim0[dim],
testbedsettings.current_testbed.rldValsOfInterest,
# TODO: let rldVals... possibly be RL-based targets
True,
outputdir, 'all', genericsettings.verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.'
% (dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
testbedsettings.current_testbed.rldValsOfInterest, True,
outputdir,
'%s' % fGroup, genericsettings.verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
testbedsettings.current_testbed.rldValsOfInterest, True,
outputdir,
'%s' % fGroup, genericsettings.verbose)
print_done()
# copy-paste from above, here for each function instead of function groups
if genericsettings.isRldOnSingleFcts:
print("ECDF graphs per function...")
# ECDFs for each function
pprldmany.all_single_functions(dictAlg,
dsList[0].isBiobjective(),
False,
sortedAlgs,
outputdir,
genericsettings.verbose,
genericsettings.two_algorithm_file_name)
print_done()
if genericsettings.isConv:
print("Convergence plots...")
ppconverrorbars.main(dictAlg,
dsList[0].isBiobjective(),
outputdir,
genericsettings.verbose,
genericsettings.two_algorithm_file_name)
print_done()
htmlFileName = os.path.join(outputdir, genericsettings.ppscatter_file_name + '.html')
if genericsettings.isScatter:
print("Scatter plots...")
ppscatter.main(dsList1, dsList0, outputdir,
verbose=genericsettings.verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\\providecommand{\\bbobppscatterlegend}[1]{',
ppscatter.figure_caption(),
'}'
])
replace_in_file(htmlFileName, '##bbobppscatterlegend##', ppscatter.figure_caption(True))
for i, alg in enumerate(args):
replace_in_file(htmlFileName, 'algorithm'
+ abc[i], str_to_latex(strip_pathname1(alg)))
print_done()
if genericsettings.isTab:
print("Generating old tables (pptable2.py)...")
dictNG0 = dsList0.dictByNoise()
dictNG1 = dsList1.dictByNoise()
for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
# split table in as many as necessary
dictFunc0 = dictNG0[nGroup].dictByFunc()
dictFunc1 = dictNG1[nGroup].dictByFunc()
funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
if len(funcs) > 24:
funcs.sort()
nbgroups = int(numpy.ceil(len(funcs) / 24.))
def split_seq(seq, nbgroups):
newseq = []
splitsize = 1.0 / nbgroups * len(seq)
for i in range(nbgroups):
newseq.append(seq[int(round(i * splitsize)):int(round((i + 1) * splitsize))])
return newseq
groups = split_seq(funcs, nbgroups)
# merge
group0 = []
group1 = []
for i, g in enumerate(groups):
tmp0 = DataSetList()
tmp1 = DataSetList()
for f in g:
tmp0.extend(dictFunc0[f])
tmp1.extend(dictFunc1[f])
group0.append(tmp0)
group1.append(tmp1)
for i, g in enumerate(zip(group0, group1)):
pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
outputdir,
'%s%d' % (nGroup, i), genericsettings.verbose)
else:
if 11 < 3: # future handling:
dictFunc0 = dsList0.dictByFunc()
dictFunc1 = dsList1.dictByFunc()
funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
funcs.sort()
# nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
# pptable2.main(dsList0, dsList1,
# testbedsettings.tabDimsOfInterest, outputdir,
# '%s' % (testbedsettings.testbedshortname), genericsettings.verbose)
else:
pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
inset.tabDimsOfInterest,
outputdir,
'%s' % (nGroup), genericsettings.verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\\providecommand{\\bbobpptablestwolegend}[1]{',
pptable2.get_table_caption(),
'}'
])
htmlFileName = os.path.join(outputdir, genericsettings.pptable2_file_name + '.html')
key = '##bbobpptablestwolegend%s##' % (testbedsettings.current_testbed.scenario)
replace_in_file(htmlFileName, '##bbobpptablestwolegend##', htmldesc.getValue(key))
replace_in_file(htmlFileName, 'algorithmAshort', algName0[0:3])
replace_in_file(htmlFileName, 'algorithmBshort', algName1[0:3])
for htmlFileName in (genericsettings.pprldistr2_file_name,
genericsettings.pptable2_file_name):
for i, alg in enumerate(args):
replace_in_file(os.path.join(outputdir, htmlFileName + '.html'),
'algorithm' + abc[i], str_to_latex(strip_pathname1(alg)))
print_done()
# The following is copied from rungenericmany.py to comply
# with the bi-objective many-algorithm LaTeX template
print("Generating new tables (pptables.py)...")
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\providecommand{\\bbobpptablesmanylegend}[2]{' +
pptables.get_table_caption() + '}'])
dictNoi = pproc.dictAlgByNoi(dictAlg)
for ng, tmpdictng in dictNoi.iteritems():
dictDim = pproc.dictAlgByDim(tmpdictng)
for d, tmpdictdim in sorted(dictDim.iteritems()):
pptables.main(
tmpdictdim,
sortedAlgs,
outputdir,
genericsettings.verbose,
([1, 20, 38] if (testbedsettings.current_testbed.name ==
testbedsettings.testbed_name_bi) else True))
print_done()
if genericsettings.isScaleUp:
print("Scaling figures...")
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
plt.rc('pdf', fonttype=42)
ppfigs.main(dictAlg,
genericsettings.ppfigs_file_name,
dsList[0].isBiobjective(),
sortedAlgs,
outputdir,
genericsettings.verbose)
plt.rcdefaults()
print_done()
ppfig.save_single_functions_html(
os.path.join(outputdir, genericsettings.two_algorithm_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.TWO,
isBiobjective=dsList0.isBiobjective(),
functionGroups=dsList0.getFuncGroups())
if (genericsettings.isFig or genericsettings.isRLDistr
or genericsettings.isTab or genericsettings.isScatter
or genericsettings.isScaleUp):
print("Output data written to folder %s" % outputdir)
plt.rcdefaults()
def pca_plot(setup,
inpar,
ofit,
prefix,
maxp=25,
pcadesc="",
addOne=True,
show=False):
""" Saves quality control plots for a PCA analysis
Parameters
----------
inpar
ofit
prefix : str
prefix for the filenames
maxp
pcadesc
addOne
Returns
-------
"""
plt.rcdefaults()
plt.rcParams['font.family'] = 'times new roman'
# Setup
method = inspect.stack()[0][3]
if not show:
outroot = qa.set_qa_filename(setup, method, prefix=prefix)
npc = inpar['npc'] + 1
pages, npp = qa.get_dimen(npc, maxp=maxp)
#
x0 = inpar['x0']
ordernum = inpar['x0in']
x0fit = inpar['x0fit']
usetrc = inpar['usetrc']
hidden = inpar['hidden']
high_fit = inpar['high_fit']
nc = np.max(ordernum[usetrc])
# Loop through all pages and plot the results
ndone = 0
for i in range(len(pages)):
plt.clf()
f, axes = plt.subplots(pages[i][1], pages[i][0])
ipx, ipy = 0, 0
if i == 0:
if pages[i][1] == 1: ind = (0, )
elif pages[i][0] == 1: ind = (0, )
else: ind = (0, 0)
axes[ind].plot(ordernum[usetrc], x0[usetrc], 'bx')
axes[ind].plot(ordernum, x0fit, 'k-')
amn, amx = np.min(x0fit), np.max(x0fit)
diff = x0[usetrc] - x0fit[usetrc]
tdiffv = np.median(diff)
mdiffv = 1.4826 * np.median(np.abs(tdiffv - diff))
amn -= 2.0 * mdiffv
amx += 2.0 * mdiffv
mval = amn - 0.15 * (amx - amn)
dmin, dmax = tdiffv - 2.0 * mdiffv, tdiffv + 2.0 * mdiffv
diff = mval + diff * 0.20 * (amx - amn) / (dmax - dmin)
wign = np.where(
np.abs(diff - np.median(diff)) < 4.0 * 1.4826 *
np.median(np.abs(diff - np.median(diff))))[0]
dmin, dmax = np.min(diff[wign]), np.max(diff[wign])
axes[ind].plot(ordernum[usetrc], diff, 'rx')
if addOne:
axes[ind].plot([0, nc + 1], [mval, mval], 'k-')
axes[ind].axis([
0, nc + 1, dmin - 0.5 * (dmax - dmin),
amx + 0.05 * (amx - amn)
])
else:
axes[ind].plot([0, nc], [mval, mval], 'k-')
axes[ind].axis([
0, nc, dmin - 0.5 * (dmax - dmin), amx + 0.05 * (amx - amn)
])
axes[ind].set_title("Mean Value")
ipx += 1
if ipx == pages[i][0]:
ipx = 0
ipy += 1
npp[0] -= 1
for j in range(npp[i]):
if pages[i][1] == 1: ind = (ipx, )
elif pages[i][0] == 1: ind = (ipy, )
else: ind = (ipy, ipx)
axes[ind].plot(ordernum[usetrc], hidden[j + ndone, :], 'bx')
axes[ind].plot(ordernum, high_fit[:, j + ndone], 'k-')
vmin, vmax = np.min(hidden[j + ndone, :]), np.max(hidden[j +
ndone, :])
if ofit[1 + j + ndone] != -1:
cmn, cmx = np.min(high_fit[:, j + ndone]), np.max(
high_fit[:, j + ndone])
diff = hidden[j + ndone, :] - high_fit[:, j + ndone][usetrc]
tdiffv = np.median(diff)
mdiffv = 1.4826 * np.median(np.abs(tdiffv - diff))
cmn -= 2.0 * mdiffv
cmx += 2.0 * mdiffv
mval = cmn - 0.15 * (cmx - cmn)
dmin, dmax = tdiffv - 2.0 * mdiffv, tdiffv + 2.0 * mdiffv
#dmin, dmax = np.min(diff), np.max(diff)
diff = mval + diff * 0.20 * (cmx - cmn) / (dmax - dmin)
wign = np.where(
np.abs(diff - np.median(diff)) < 4.0 * 1.4826 *
np.median(np.abs(diff - np.median(diff))))[0]
dmin, dmax = np.min(diff[wign]), np.max(diff[wign])
#vmin, vmax = np.min(hidden[j+ndone,:][wign]), np.max(hidden[j+ndone,:][wign])
axes[ind].plot(ordernum[usetrc], diff, 'rx')
axes[ind].plot([0, 1 + nc], [mval, mval], 'k-')
# ymin = np.min([(3.0*dmin-dmax)/2.0,vmin-0.1*(vmax-dmin),dmin-0.1*(vmax-dmin)])
# ymax = np.max([np.max(high_fit[:,j+ndone]),vmax+0.1*(vmax-dmin),dmax+0.1*(vmax-dmin)])
ymin = dmin - 0.5 * (dmax - dmin)
ymax = cmx + 0.05 * (cmx - cmn)
if addOne: axes[ind].axis([0, nc + 1, ymin, ymax])
else: axes[ind].axis([0, nc, ymin, ymax])
else:
if addOne:
axes[ind].axis([
0, nc + 1, vmin - 0.1 * (vmax - vmin),
vmax + 0.1 * (vmax - vmin)
])
else:
axes[ind].axis([
0, nc, vmin - 0.1 * (vmax - vmin),
vmax + 0.1 * (vmax - vmin)
])
axes[ind].set_title("PC {0:d}".format(j + ndone))
axes[ind].tick_params(labelsize=8)
ipx += 1
if ipx == pages[i][0]:
ipx = 0
ipy += 1
if i == 0: npp[0] = npp[0] + 1
# Delete the unnecessary axes
for j in range(npp[i], axes.size):
if pages[i][1] == 1: ind = (ipx, )
elif pages[i][0] == 1: ind = (ipy, )
else: ind = (ipy, ipx)
f.delaxes(axes[ind])
ipx += 1
if ipx == pages[i][0]:
ipx = 0
ipy += 1
ndone += npp[i]
# Save the figure
if pages[i][1] == 1 or pages[i][0] == 1: ypngsiz = 11.0 / axes.size
else: ypngsiz = 11.0 * axes.shape[0] / axes.shape[1]
f.set_size_inches(11.0, ypngsiz)
if pcadesc != "":
pgtxt = ""
if len(pages) != 1:
pgtxt = ", page {0:d}/{1:d}".format(i + 1, len(pages))
f.suptitle(pcadesc + pgtxt, y=1.02, size=16)
f.tight_layout()
if show:
plt.show()
else:
outfile = outroot + '{:02d}.png'.format(i)
f.savefig(outfile, dpi=200)
plt.close()
f.clf()
del f
plt.rcdefaults()
return
def make_comparison_plots(forecaster_dict, path=''):
import matplotlib.pyplot as plt
plt.rcdefaults()
import numpy as np
forecasters = [n for n, f in forecaster_dict.items()]
y_pos = np.arange(len(forecasters))
all_danger_levels = [
f.danger_levels_avg for n, f in forecaster_dict.items()
]
all_danger_levels_std = [
f.danger_levels_std for n, f in forecaster_dict.items()
]
all_problems_pr_warning = [
f.problems_pr_warning_avg for n, f in forecaster_dict.items()
]
all_problems_pr_warning_std = [
f.problems_pr_warning_std for n, f in forecaster_dict.items()
]
all_nowcast_lengths = [
f.nowcast_lengths_avg for n, f in forecaster_dict.items()
]
all_nowcast_lengths_std = [
f.nowcast_lengths_std for n, f in forecaster_dict.items()
]
all_forecast_lengths = [
f.forecast_lengths_avg for n, f in forecaster_dict.items()
]
all_forecast_lengths_std = [
f.forecast_lengths_std for n, f in forecaster_dict.items()
]
# all danger levels
plt.clf()
plt.figure(figsize=(12, 8))
plt.barh(y_pos,
all_danger_levels,
align='center',
alpha=0.5,
color='pink',
xerr=all_danger_levels_std,
error_kw=dict(ecolor='k', lw=0.25, capsize=1.5, capthick=1.5))
plt.yticks(y_pos, forecasters, rotation=0)
plt.xlabel('Gjennomsnittlig faregrad')
plt.xlim(0., 4.5)
plt.axvline(forecaster_dict['Ragnar@NVE'].danger_levels_all_avg,
color='k',
linestyle='dashed',
linewidth=3)
plt.title('Sammenlignet varselt faregrad sesongen 2016/17')
fig = plt.gcf()
fig.subplots_adjust(left=0.2)
plt.savefig('{0}all_danger_201617.png'.format(path))
# all problems pr warning
plt.clf()
plt.figure(figsize=(12, 8))
plt.barh(y_pos,
all_problems_pr_warning,
align='center',
alpha=0.5,
color='pink',
xerr=all_problems_pr_warning_std,
error_kw=dict(ecolor='k', lw=0.25, capsize=1.5, capthick=1.5))
plt.yticks(y_pos, forecasters, rotation=0)
plt.xlabel('Snitt antall skredproblemer')
plt.xlim(0.5, 3.)
plt.axvline(forecaster_dict['Ragnar@NVE'].problems_pr_warning_all_avg,
color='k',
linestyle='dashed',
linewidth=3)
plt.title('Antall skredproblemer pr varsel sesongen 2016/17')
fig = plt.gcf()
fig.subplots_adjust(left=0.2)
plt.savefig('{0}all_problems_pr_warning_201617.png'.format(path))
# all nowcast lengths
plt.clf()
plt.figure(figsize=(12, 8))
plt.barh(y_pos,
all_nowcast_lengths,
align='center',
alpha=0.5,
color='b',
xerr=all_nowcast_lengths_std,
error_kw=dict(ecolor='k', lw=0.25, capsize=1.5, capthick=1.5))
plt.yticks(y_pos, forecasters, rotation=0)
plt.xlabel('Snitt tegn paa naasituasjon')
plt.xlim(0., 1024.)
plt.axvline(forecaster_dict['Ragnar@NVE'].nowcast_lengths_all_avg,
color='k',
linestyle='dashed',
linewidth=3)
plt.title('Antall tegn brukt i naasituasjonen sesongen 2016/17')
fig = plt.gcf()
fig.subplots_adjust(left=0.2)
plt.savefig('{0}all_nowcast_lengths_201617.png'.format(path))
# all forecast lengths
plt.clf()
plt.figure(figsize=(12, 8))
plt.barh(y_pos,
all_forecast_lengths,
align='center',
alpha=0.5,
color='b',
xerr=all_forecast_lengths_std,
error_kw=dict(ecolor='k', lw=0.25, capsize=1.5, capthick=1.5))
plt.yticks(y_pos, forecasters, rotation=0)
plt.xlabel('Snitt tegn paa varselet')
plt.xlim(0., 1024.)
plt.axvline(forecaster_dict['Ragnar@NVE'].forecast_lengths_all_avg,
color='k',
linestyle='dashed',
linewidth=3)
plt.title('Antall tegn brukt i varselet sesongen 2016/17')
fig = plt.gcf()
fig.subplots_adjust(left=0.2)
plt.savefig('{0}all_forecast_lengths_201617.png'.format(path))
return
import nltk
from nltk import word_tokenize
import linecache
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
from nltk.corpus import stopwords
nltk.download('stopwords')
import matplotlib.pyplot as plot
plot.rcdefaults()
import numpy as np
linecounter = 0
pizza_count = 0
chocolate_count = 0
food_count = 0
f = open('Extracted_tokens.txt','a')
f1 = open('Extracted_urls.txt','a')
punctuations = "?:!.,;"
stop_words = set(stopwords.words('english'))
with open('SBU_captioned_photo_dataset_captions.txt','r') as file:
for line in file:
linecounter = linecounter + 1
tokens = word_tokenize(line)
for word in tokens:
if word in punctuations:
tokens.remove(word)
if word in stop_words:
tokens.remove(word)
def CGmapMethylInBins(fn,
coverage,
coverageXY,
step,
CTX,
filetype='png',
prefix="",
title="",
fH=4.0,
fW=8.0):
try:
if fn:
if fn.endswith(".gz"):
IN = gzip.open(fn, 'rb')
else:
IN = open(fn, 'r')
#
else:
IN = sys.stdin
#
except IOError:
print "\n[Error]:\n\t File cannot be open: ", fn
exit(-1)
#
line = IN.readline()
posL = 1
posR = step
preChr = ""
bin_list = []
mC_list = []
chr_list = []
chr_start_list = []
chr_end_list = []
#
while line:
try:
chr, nuc, pos, pattern, dinuc, methyl, MC, NC = line.strip().split(
)
except ValueError:
print(
"\n[Error]:\n\t File [ %s ] may have wrong number of columns."
% fn)
exit(-1)
#
pos = int(pos)
MC = int(MC)
NC = int(NC)
# check if line could be filtered by context
if CTX not in ["", "C"]:
if CTX in ["CG", "CHG", "CHH"]:
if pattern != CTX:
line = IN.readline()
continue
#
elif CTX in ["CA", "CC", "CT"]:
if dinuc != CTX:
line = IN.readline()
continue
#
elif CTX == "CH":
if dinuc not in ["CHG", "CHH"]:
line = IN.readline()
continue
#
elif CTX == "CW":
if dinuc not in ["CA", "CC"]:
line = IN.readline()
continue
#
else:
line = IN.readline()
continue
#
#
methyl = float(MC) / NC
if (NC >= coverage) or ((chr == "chrX" or chr == "chrY"
or chr == "ChrX" or chr == "ChrY") and
(NC >= coverageXY)):
if chr != preChr:
if bin_list == []:
if preChr != "":
print("%s\t%d\t%d\tna" % (preChr, posL, posR))
mC_list.append(float('nan'))
#
else:
chr_end_list.append(len(mC_list))
mean_mC = average(bin_list)
print("%s\t%d\t%d\t%.4f" % (preChr, posL, posR, mean_mC))
mC_list.append(mean_mC)
bin_list = []
#
chr_list.append(chr)
chr_start_list.append(len(mC_list))
posL = 1
posR = step
preChr = chr
#
if pos <= posR:
bin_list.append(methyl)
else:
if bin_list == []:
print("%s\t%d\t%d\tna" % (preChr, posL, posR))
mC_list.append(float('nan'))
else:
mean_mC = average(bin_list)
print("%s\t%d\t%d\t%.4f" % (preChr, posL, posR, mean_mC))
mC_list.append(mean_mC)
#
bin_list = []
posL = posR + 1
posR += step
#
#
line = IN.readline()
#
if bin_list == []:
if preChr != "":
print("%s\t%d\t%d\tna" % (preChr, posL, posR))
mC_list.append(float('nan'))
#
else:
mean_mC = average(bin_list)
print("%s\t%d\t%d\t%.4f" % (preChr, posL, posR, mean_mC))
mC_list.append(mean_mC)
#
chr_end_list.append(len(mC_list))
if IN is not sys.stdin:
IN.close()
#
#
if filetype in ['png', 'eps', 'pdf']:
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.rcdefaults()
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
# ===
chr_mid_list = [(s + e) / 2
for s, e in zip(chr_start_list, chr_end_list)]
plt.figure(figsize=(fW, fH))
MaxMC = NanMax(mC_list) * 1.1 + NanMin(mC_list)
plt.ylim([0, MaxMC])
plt.plot(range(0, len(mC_list), 1), mC_list, 'b-')
for x in chr_start_list[1:]:
plt.plot([x, x], [-MaxMC, MaxMC], 'k-')
#
plt.xticks(chr_mid_list, chr_list, rotation=25)
plt.ylabel("Average methylation level")
plt.title(title)
#
if prefix != "":
prefix = prefix + "."
#
plt.savefig(prefix + "MethInBins." + filetype, format=filetype)
plt.clf()
def quality_visual(ax=None):
"""Visualizing Multiple Measures of Forecast Quality
refer to:
Roebber, P.J., 2009: Visualizing Multiple Measures of Forecast Quality.
Wea. Forecasting, 24, 601-608, https://doi.org/10.1175/2008WAF2222159.1
Keyword Arguments:
ax {matplotlib.axes} -- matplotlib axes instance (default: {None})
"""
# set up figure
if ax is None:
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(111)
# modifying rc settings
plt.rc('font', size=20)
plt.rc('axes', linewidth=3)
plt.rc('xtick.major', size=14, width=3)
plt.rc('ytick.major', size=14, width=3)
plt.rc('xtick.minor', size=8, width=1)
plt.rc('ytick.minor', size=8, width=1)
# define SR
SR = np.arange(0.0, 1.01, 0.01)
# draw BS lines
BSs = [0.3, 0.5, 0.8, 1.0, 1.3, 1.5, 2.0, 3.0, 5.0, 10.0]
for bs in BSs:
ax.plot(SR,
bs * SR,
color="black",
linewidth=1,
linestyle='--',
label="BIAS")
if bs < 1.0:
ax.text(1.02, bs, str(bs), fontsize=16)
elif bs > 1.0:
ax.text(1.0 - (bs - 1) / bs,
1.02,
str(bs),
ha='center',
fontsize=16)
else:
ax.text(1.02, 1.02, '1.0', fontsize=16)
# draw CSI line
CSIs = np.arange(0.1, 1.0, 0.1)
x_pos = [0.5, 0.576, 0.652, 0.728, 0.804, 0.88, 0.88, 0.93, 0.97]
for i, csi in enumerate(CSIs):
pod = SR / (SR / csi + SR - 1.0)
pod[pod < 0] = np.nan
ax.plot(SR, pod, color="black", linewidth=1, label="CSI")
ax.text(x_pos[i],
x_pos[i] / (x_pos[i] / csi + x_pos[i] - 1.0),
"{:.1f}".format(csi),
backgroundcolor="white",
fontsize=12)
# set axis style
majorLocator = MultipleLocator(0.1)
minorLocator = MultipleLocator(0.02)
ax.xaxis.set_major_locator(majorLocator)
ax.xaxis.set_minor_locator(minorLocator)
ax.yaxis.set_major_locator(majorLocator)
ax.yaxis.set_minor_locator(minorLocator)
ax.set_xlim(-0.01, 1.01)
ax.set_ylim(-0.01, 1.01)
ax.set_xlabel('Success Ratio (1-FAR)')
ax.set_ylabel('Probability of Detection (POD)')
# restore default rc settings
plt.rcdefaults()
return ax
def plot_spectra_grid(hi_velocity_axis_list=None, hi_spectrum_list=None,
hi_vel_range_list=None, co_spectrum_list=None,
co_velocity_axis_list=None, hi_std_list=None, title=None, limits=None,
savedir='./', filename=None, show=True, spectra_names='',):
# Import external modules
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from mpl_toolkits.axes_grid1 import ImageGrid
import pyfits as pf
import matplotlib.pyplot as plt
import pywcsgrid2 as wcs
import pywcs
from pylab import cm # colormaps
from matplotlib.patches import Polygon
# Set up plot aesthetics
# ----------------------
plt.close;plt.clf()
plt.rcdefaults()
# Color map
cmap = plt.cm.gnuplot
# Color cycle, grabs colors from cmap
color_cycle = [cmap(i) for i in np.linspace(0, 0.8, 2)]
#font_scale = 9
font_scale = 15
line_weight = 600
font_weight = 600
params = {
'axes.color_cycle': color_cycle, # colors of different plots
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
#'axes.weight': line_weight,
'axes.linewidth': 1.2,
'axes.labelweight': font_weight,
'legend.fontsize': font_scale*3/4,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': font_weight,
'font.serif': 'computer modern roman',
'text.fontsize': font_scale,
'text.usetex': True,
'text.latex.preamble': r'\usepackage[T1]{fontenc}',
#'font.family': 'sans-serif',
'figure.figsize': (7.3, 7.3),
'figure.dpi': 600,
'backend' : 'pdf',
#'figure.titlesize': font_scale,
}
plt.rcParams.update(params)
pgf_with_pdflatex = {
"pgf.texsystem": "pdflatex",
"pgf.preamble": [
r"\usepackage[utf8x]{inputenc}",
r"\usepackage[T1]{fontenc}",
r"\usepackage{cmbright}",
]
}
plt.rcParams.update(pgf_with_pdflatex)
# Create figure instance
fig = plt.figure()
# Determine number of plots on each axis
n = int(np.ceil(len(hi_velocity_axis_list)**0.5))
imagegrid = ImageGrid(fig, (1,1,1),
nrows_ncols=(3, 2),
ngrids=len(hi_velocity_axis_list),
axes_pad=0,
aspect=False,
label_mode='L',
share_all=True)
for i in xrange(len(hi_velocity_axis_list)):
# create axes
ax = imagegrid[i]
ax.plot(hi_velocity_axis_list[i],
hi_spectrum_list[i],
#color='k',
linestyle='--',
label='Median HI',
drawstyle = 'steps-mid'
)
if hi_std_list is not None:
ax.plot(hi_velocity_axis_list[i],
hi_std_list[i],
#color='k',
linestyle='-.',
label=r'$\sigma_{\rm HI}$',
drawstyle='steps-mid'
)
if co_velocity_axis_list is not None:
ax.plot(co_velocity_axis_list[i],
co_spectrum_list[i] * 50.,
#color='k',
label=r'Median $^{12}$CO $\times$ 50',
drawstyle = 'steps-mid'
)
spectra_names[i] = spectra_names[i].replace('1', ' 1')
spectra_names[i] = spectra_names[i].replace('2', ' 2')
if 0:
ax.annotate(spectra_names[i].capitalize(),
xytext=(0.1, 0.9),
xy=(0.1, 0.9),
textcoords='axes fraction',
xycoords='axes fraction',
color='k'
)
else:
ax.annotate(spectra_names[i].capitalize(),
xytext=(0.96, 0.9),
xy=(0.96, 0.9),
textcoords='axes fraction',
xycoords='axes fraction',
size=font_scale,
color='k',
bbox=dict(boxstyle='square',
facecolor='w',
alpha=1),
horizontalalignment='right',
verticalalignment='top',
)
ax.axvspan(hi_vel_range_list[i][0],
hi_vel_range_list[i][1],
alpha=0.3,
color='k',
)
ax.set_xlabel('Velocity [km/s]')
ax.set_ylabel(r'T$_b$ [K]')
if i == 0:
ax.legend(loc='upper left')
# plot limits
if limits is not None:
ax.set_xlim(limits[0],limits[1])
ax.set_ylim(limits[2],limits[3])
if 0:
if co_velocity_axis_list is not None:
# Single legend
ax.legend(bbox_to_anchor=(1.5, 0.2),
loc='lower right',
borderaxespad=0.)
if title is not None:
fig.suptitle(title, fontsize=fontScale)
if filename is not None:
plt.savefig(savedir + filename, bbox_inches='tight', dpi=600)
if show:
fig.show()
def pltdefault():
global figdir
plt.rcdefaults()
figdir = datadir + 'samoa/WATERSHED_ANALYSIS/GoodFigures/rawfigoutput/'
return
async def frequency(self, ctx, *args):
logger.info(
"[Administration frequency()] frequency command detected from " +
str(ctx.message.author) + " with arguments [" + str(args) + "]")
if len(args) == 0:
await ctx.send("please specify which columns you want to count=" +
str(list(self.get_column_headers_from_database())))
return
else:
dicResult = self.determineXYFrequency(self.connectToDatabase(),
args)
dicResult = sorted(dicResult.items(), key=lambda kv: kv[1])
logger.info("[Administration frequency()] sorted dicResults by value")
if len(dicResult) <= 50:
logger.info(
"[Administration frequency()] dicResults's length is <= 50")
labels = [i[0] for i in dicResult]
numbers = [i[1] for i in dicResult]
plt.rcdefaults()
fig, ax = plt.subplots()
y_pos = np.arange(len(labels))
for i, v in enumerate(numbers):
ax.text(v, i + .25, str(v), color='blue', fontweight='bold')
ax.barh(y_pos, numbers, align='center', color='green')
ax.set_yticks(y_pos)
ax.set_yticklabels(labels)
ax.invert_yaxis() # labels read top-to-bottom
if len(args) > 1:
title = '_'.join(str(arg) for arg in args[:len(args) - 1])
title += "_" + args[len(args) - 1]
else:
title = args[0]
ax.set_title("How may times each " + title +
" appears in the database")
fig.set_size_inches(18.5, 10.5)
fig.savefig('image.png')
logger.info("[Administration frequency()] graph created and saved")
plt.close(fig)
await ctx.send(file=discord.File('image.png'))
logger.info(
"[Administration frequency()] graph image file has been sent")
else:
logger.info(
"[Administration frequency()] dicResults's length is > 50")
numberOfPages = int(len(dicResult) / 50)
if len(dicResult) % 50 != 0:
numberOfPages += 1
numOfBarsPerPage = int(len(dicResult) / numberOfPages) + 1
firstIndex, lastIndex = 0, numOfBarsPerPage - 1
msg = None
currentPage = 0
while firstIndex < len(dicResult):
logger.info(
"[Administration frequency()] creating a graph with entries "
+ str(firstIndex) + " to " + str(lastIndex))
toReact = ['⏪', '⏩', '✅']
labels = [i[0] for i in dicResult][firstIndex:lastIndex]
numbers = [i[1] for i in dicResult][firstIndex:lastIndex]
plt.rcdefaults()
fig, ax = plt.subplots()
y_pos = np.arange(len(labels))
for i, v in enumerate(numbers):
ax.text(v,
i + .25,
str(v),
color='blue',
fontweight='bold')
ax.barh(y_pos, numbers, align='center', color='green')
ax.set_yticks(y_pos)
ax.set_yticklabels(labels)
ax.invert_yaxis() # labels read top-to-bottom
ax.set_xlabel("Page " + str(currentPage) + "/" +
str(numberOfPages - 1))
if len(args) > 1:
title = '_'.join(str(arg) for arg in args[:len(args) - 1])
title += "_" + args[len(args) - 1]
else:
title = args[0]
ax.set_title("How may times each " + title +
" appears in the database")
fig.set_size_inches(18.5, 10.5)
fig.savefig('image.png')
logger.info(
"[Administration frequency()] graph created and saved")
plt.close(fig)
if msg is None:
msg = await ctx.send(file=discord.File('image.png'))
else:
await msg.delete()
msg = await ctx.send(file=discord.File('image.png'))
for reaction in toReact:
await msg.add_reaction(reaction)
def checkReaction(reaction, user):
if not user.bot: ##just making sure the bot doesnt take its own reactions
##into consideration
e = str(reaction.emoji)
logger.info(
"[numOfBarsPerPage frequency()] reaction " + e +
" detected from " + str(user))
return e.startswith(('⏪', '⏩', '✅'))
logger.info(
"[Administration frequency()] graph image file has been sent"
)
userReacted = False
while userReacted == False:
try:
userReacted = await self.bot.wait_for(
'reaction_add', timeout=20, check=checkReaction)
except asyncio.TimeoutError:
logger.info(
"[Administration frequency()] timed out waiting for the user's reaction."
)
if userReacted != False:
if '⏪' == userReacted[0].emoji:
firstIndex -= numOfBarsPerPage
lastIndex -= numOfBarsPerPage
currentPage -= 1
if firstIndex < 0:
firstIndex, lastIndex = numOfBarsPerPage * 3, numOfBarsPerPage * 4
currentPage = numberOfPages - 1
logger.info(
"[Administration frequency()] user indicates they want to go back to page "
+ str(currentPage))
elif '⏩' == userReacted[0].emoji:
firstIndex += numOfBarsPerPage
lastIndex += numOfBarsPerPage
currentPage += 1
if firstIndex > len(dicResult):
firstIndex, lastIndex = 0, numOfBarsPerPage
currentPage = 0
logger.info(
"[Administration frequency()] user indicates they want to go to page "
+ str(currentPage))
elif '✅' == userReacted[0].emoji:
logger.info(
"[Administration frequency()] user indicates they are done with the roles command, deleting roles message"
)
await msg.delete()
return
else:
logger.info(
"[Administration frequency()] deleting message")
await msg.delete()
return
logger.info(
"[Administration frequency()] updating firstIndex and lastIndex to "
+ str(firstIndex) + " and " + str(lastIndex) +
" respectively")
ax.set_ylim([-30, 10]) # Setting the limit for y axis
# Creating the data to be displayed
data = np.ones(100) # Creating the data to be displayed
data[70:] -= np.arange(30) # Creating the data to be displayed
# Preparing the text to be displayed/annotated
# plt.annotate('This is my\n first\n XKCD Plot', xy(70, 1), arrowprops=dict(arrowstyle='->'), xytext=(-15,10))
plt.plot(data) # Plotting the graph with the data
plt.xlabel('Time') # Display the label for X-Axis
plt.xlabel('My Practice') # Display the label for Y-Axis
# Pie Chart- Set 1
plt.rcdefaults() # Return to default as we had set the plotting to XKCD mode in the previous example
values = [12, 55, 4, 32, 14] # Setting the percentage values to be shown on the pie chart
colors = ['r', 'g', 'b', 'c', 'm'] # Setting the color values to be shown on the pie chart
explode = [0, 0, 0.2, 0, 0] # Setting the percentage value to explode and come out
labels = ['India', 'United States', 'Russia', 'China', 'Europe'] # Setting the label values to be shown on the pie chart
plt.pie(values, colors = colors, labels = labels, explode = explode) # Creating the pie chart
plt.title('Pie Chart') # Setting the pie chart title to be shown on the pie chart
plt.show() # Display the plot
# Pie Chart- Set 2
plt.rcdefaults() # Return to default as we had set the plotting to XKCD mode in the previous example
values = [12, 55, 4, 32, 14] # Setting the percentage values to be shown on the pie chart
colors = ['r', 'g', 'b', 'c', 'm'] # Setting the color values to be shown on the pie chart
explode = [0.5, 0, 0.2, 0, 0] # Setting the percentage value to explode and come out
labels = ['India', 'United States', 'Russia', 'China', 'Europe'] # Setting the label values to be shown on the pie chart
plt.pie(values, colors = colors, labels = labels, explode = explode) # Creating the pie chart
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for one algorithm (``*article.tex`` or ``*1*.tex``).
The used template file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder created
by this routine.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--rld-single-fcts
generate also runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--svg
generate also the svg figures which are used in html files
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not '--help' in argv and not 'h' in argv:
print 'not enough input arguments given'
print 'cave: the following options also need an argument:'
print [o for o in genericsettings.longoptlist if o[-1] == '=']
print 'options given:'
print opts
print 'try --help for help'
sys.exit()
# Process options
outputdir = genericsettings.outputdir
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
genericsettings.isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--fig-only":
genericsettings.isTab = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--rld-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isLogLoss = False
elif o == "--los-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isRLDistr = False
elif o == "--crafting-effort":
try:
genericsettings.inputCrE = float(a)
except ValueError:
raise Usage('Expect a valid float for flag crafting-effort.')
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--conv":
genericsettings.isConv = True
elif o == "--rld-single-fcts":
genericsettings.isRldOnSingleFcts = True
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
genericsettings.isExpensive = False
elif o == "--svg":
genericsettings.generate_svg_files = True
elif o == "--sca-only":
warnings.warn("option --sca-only will have no effect with rungeneric1.py")
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if genericsettings.inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif genericsettings.inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif genericsettings.inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % genericsettings.inputsettings
+ 'argument for input flag "--settings".')
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config()
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
finally:
fp.close()
if (not genericsettings.verbose):
warnings.simplefilter('module')
# warnings.simplefilter('ignore')
#get directory name if outputdir is a archive file
algfolder = findfiles.get_output_directory_subfolder(args[0])
outputdir = os.path.join(outputdir, algfolder)
print ("Post-processing (1): will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, genericsettings.verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, genericsettings.verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dsList = dsList.dictByNoise().get('nzall', DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dsList = dsList.dictByNoise().get('noiselessall', DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
from bbob_pproc import config
config.target_values(genericsettings.isExpensive, dict_max_fun_evals)
config.config()
if (genericsettings.verbose):
for i in dsList:
if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) !=
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % str(dictAlg.keys()) +
'will be processed together.')
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print 'Folder %s was created.' % (outputdir)
if genericsettings.isPickled:
dsList.pickle(verbose=genericsettings.verbose)
if genericsettings.isConv:
ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)
if genericsettings.isFig:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
plt.rc('pdf', fonttype = 42)
ppfigdim.main(dsList, ppfigdim.values_of_interest,
outputdir, genericsettings.verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc('pdf', fonttype = 42)
if genericsettings.isTab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest,
outputdir, noise, genericsettings.verbose)
print_done()
if genericsettings.isRLDistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True,
outputdir, 'all', genericsettings.verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True,
outputdir,
'%s' % noise, genericsettings.verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, True,
outputdir,
'%s' % fGroup, genericsettings.verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
if genericsettings.isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
pprldmany.all_single_functions(dictAlg, None,
outputdir,
genericsettings.verbose)
print_done()
if genericsettings.isLogLoss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value "
+ "for %s testbed:\n CrE = " % testbed)
CrE = genericsettings.inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = '%s' % ng
pplogloss.main(sliceDim, CrE, True,
outputdir, info,
verbose=genericsettings.verbose)
pplogloss.generateTable(sliceDim, CrE,
outputdir, info,
verbose=genericsettings.verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
info = '%s' % fGroup
pplogloss.main(sliceFuncGroup, CrE, True,
outputdir, info,
verbose=genericsettings.verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
html_file = os.path.join(outputdir, genericsettings.single_algorithm_file_name + '.html')
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglosstablecaption}[1]{',
pplogloss.table_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglossfigurecaption}[1]{',
pplogloss.figure_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpprldistrlegend}[1]{',
pprldistr.caption_single(np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems()])), # depends on the config setting, should depend on maxfevals
'}'])
replace_in_file(html_file, r'TOBEREPLACED', 'D, '.join([str(i) for i in pprldistr.single_runlength_factors[:6]]) + 'D,…')
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobppfigdimlegend}[1]{',
ppfigdim.scaling_figure_caption(),
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpptablecaption}[1]{',
pptable.table_caption,
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algfolder}{' + algfolder + '/}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algname}{' +
(str_to_latex(strip_pathname1(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'])
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
def main(argv=None):
r"""Main routine for post-processing the data of multiple algorithms.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for comparing multiple algorithms (``*many.tex`` or ``*3*.tex``).
The used template file needs to be edited so that the commands
``\bbobdatapath`` points to the output folder created by this routine.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
provided, sys.argv is accessed.
*argv* must list folders containing BBOB data files.
The name of these folders will be used in the output figures and
tables to designate the algorithms. Therefore you should name the
folders with differentiating names.
Furthermore, argv can begin with facultative option flags listed
below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out operations, warnings.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdatamany`) to
:file:`OUTPUTDIR`.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale" or "color".
The default setting is "color".
--tab-only, --rld-only, --fig-only
these options can be used to output respectively the
comparison TeX tables, the run lengths distributions or the
figures of aRT/dim vs dim only. A combination of any two or
more of these options results in no output.
--no-rld-single-fcts
do not generate runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets.
--no-svg
do not generate the svg figures which are used in html files
-
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungenericmany.py interface from the command line::
$ python -m cocopp.rungenericmany -v AMALGAM BFGS BIPOP-CMA-ES
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m cocopp.rungenericmany -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import cocopp
>> cocopp.rungenericmany.main('-o outputfolder folder1 folder2'.split())
This will execute the post-processing on the data found in
:file:`folder1` and :file:`folder2`.
The ``-o`` option changes the output folder from the default to
:file:`outputfolder`.
* Generate post-processing data for some algorithms with figures in
shades of gray::
$ python rungenericmany.py --settings grayscale NEWUOA NELDER LSSTEP
"""
if argv is None:
argv = sys.argv[1:]
try:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist,
genericsettings.longoptlist)
except getopt.error as msg:
raise Usage(msg)
if not args:
usage()
sys.exit()
# Process options
outputdir = genericsettings.outputdir
prepare_scatter = genericsettings.isScatter
prepare_RLDistr = genericsettings.isRLDistr
prepare_figures = genericsettings.isFig
prepare_tables = genericsettings.isTab
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
# The next 3 are for testing purpose
elif o == "--no-rld-single-fcts":
genericsettings.isRldOnSingleFcts = False
elif o == "--tab-only":
prepare_RLDistr = False
prepare_figures = False
prepare_scatter = False
elif o == "--rld-only":
prepare_tables = False
prepare_figures = False
prepare_scatter = False
elif o == "--fig-only":
prepare_RLDistr = False
prepare_tables = False
prepare_scatter = False
elif o == "--sca-only":
prepare_figures = False
prepare_RLDistr = False
prepare_tables = False
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--no-svg":
genericsettings.generate_svg_files = False
elif o == "--los-only":
warnings.warn(
"option --los-only will have no effect with rungenericmany.py"
)
elif o == "--crafting-effort=":
warnings.warn(
"option --crafting-effort will have no effect with rungenericmany.py"
)
elif o in ("-p", "--pickle"):
warnings.warn(
"option --pickle will have no effect with rungenericmany.py"
)
else:
assert False, "unhandled option"
# from cocopp import bbob2010 as inset # input settings
# TODO: conditional imports are NOT the way to go here
if genericsettings.inputsettings == "color":
from . import config, genericsettings as inset # input settings
config.config()
elif genericsettings.inputsettings == "grayscale":
# this settings strategy (by proving different settings files) is problematic,
# because it means copy-paste of the settings
# file and future changes have a great chance to make the pasted files incompatible
# as has most likely happened with grayscalesettings:
from . import config, grayscalesettings as inset # input settings
# better would be just adjust the previous settings, as config is doing it,
# so a config_grayscalesettings.py module seems the better approach to go
elif genericsettings.inputsettings == "black-white":
from . import config, bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' %
genericsettings.inputsettings +
'argument for input flag "--settings".')
raise Usage(txt)
if not genericsettings.verbose:
warnings.filterwarnings('module', '.*', Warning,
'.*') # same warning just once
#warnings.simplefilter('ignore') # that is bad, but otherwise to many warnings appear
config.config_target_values_setting(
genericsettings.isExpensive,
genericsettings.runlength_based_targets)
except Usage as err:
print(err.msg, file=sys.stderr)
print("for help use -h or --help", file=sys.stderr)
return 2
if 1 < 3:
print("\nPost-processing (2+)")
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print('Folder %s was created.' % outputdir)
latex_commands_file = os.path.join(outputdir, 'cocopp_commands.tex')
# prepend the algorithm name command to the tex-command file
lines = []
# first prepare list of sorted algorithm names as displayed
algs = []
for alg in args:
algs.append(str_to_latex(strip_pathname1(alg)))
algs.sort()
# now ready for writing the sorted algorithms as \providecommand in tex-command file
for i, alg in enumerate(algs):
lines.append('\\providecommand{\\algorithm' + pptex.numtotext(i) +
'}{' + str_to_latex(strip_pathname1(alg)) + '}')
prepend_to_file(
latex_commands_file, lines, 5000,
'bbob_proc_commands.tex truncated, consider removing ' +
'the file before the text run')
print(" loading data...")
dsList, sortedAlgs, dictAlg = processInputArgs(args, True)
# filter data set lists to be compliant with all suites
# (useful right now only for bbob-biobj and bbob-biobj-ext data)
dsList = testbedsettings.current_testbed.filter(dsList)
dictAlg = testbedsettings.current_testbed.filter(dictAlg)
if not dsList:
sys.exit()
algorithm_folder = findfiles.get_output_directory_sub_folder(
genericsettings.foreground_algorithm_list)
prepend_to_file(
latex_commands_file,
['\\providecommand{\\algsfolder}{' + algorithm_folder + '/}'])
many_algorithms_output = os.path.join(outputdir, algorithm_folder)
print(" Will generate output data in folder %s" %
many_algorithms_output)
print(" this might take several minutes.")
if not os.path.exists(many_algorithms_output):
os.makedirs(many_algorithms_output)
if genericsettings.verbose:
print('Folder %s was created.' % many_algorithms_output)
for i in dictAlg:
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'nzall', DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'noiselessall', DataSetList())
# set target values
from . import config
config.config_target_values_setting(
genericsettings.isExpensive,
genericsettings.runlength_based_targets)
config.config(dsList[0].testbed_name)
for i in dsList:
if i.dim not in testbedsettings.current_testbed.dimensions_to_display:
continue
# check whether current set of instances correspond to correct
# setting of a BBOB workshop and issue a warning otherwise:
curr_instances = (dict((j, i.instancenumbers.count(j))
for j in set(i.instancenumbers)))
correct = False
for instance_set_of_interest in inset.instancesOfInterest:
if curr_instances == instance_set_of_interest:
correct = True
try:
last_incorrect_instances
except:
last_incorrect_instances = None
if not correct and curr_instances != last_incorrect_instances:
warnings.warn('The data of %s ' % i +
'(and possibly further data) do not ' +
'list the correct instances ' +
'of function F%d.' % i.funcId)
last_incorrect_instances = curr_instances
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc('pdf', fonttype=42)
ppfig.copy_js_files(many_algorithms_output)
ppfig.save_single_functions_html(
os.path.join(many_algorithms_output,
genericsettings.many_algorithm_file_name),
'', # algorithms names are clearly visible in the figure
htmlPage=ppfig.HtmlPage.MANY,
function_groups=dictAlg[sortedAlgs[0]].getFuncGroups())
ppfig.save_single_functions_html(
os.path.join(many_algorithms_output,
genericsettings.ppfigs_file_name),
'', # algorithms names are clearly visible in the figure
htmlPage=ppfig.HtmlPage.PPFIGS,
function_groups=dictAlg[sortedAlgs[0]].getFuncGroups(),
parentFileName=genericsettings.many_algorithm_file_name)
dimensions = sorted(pproc.dictAlgByDim(dictAlg))
ppfig.save_single_functions_html(
os.path.join(many_algorithms_output,
genericsettings.pptables_file_name),
'', # algorithms names are clearly visible in the figure
dimensions=dimensions,
htmlPage=ppfig.HtmlPage.PPTABLES,
function_groups=dictAlg[sortedAlgs[0]].getFuncGroups(),
parentFileName=genericsettings.many_algorithm_file_name)
# empirical cumulative distribution functions (ECDFs) aka Data profiles
if prepare_RLDistr:
config.config(dsList[0].testbed_name)
if len(genericsettings.foreground_algorithm_list) == 2:
print("ECDF runlength ratio graphs...")
ds_list0 = dictAlg[sortedAlgs[0]]
dict_fun0 = ds_list0.dictByNoise()
ds_list1 = dictAlg[sortedAlgs[1]]
dict_fun1 = ds_list1.dictByNoise()
if len(dict_fun0) > 1 or len(dict_fun1) > 1:
warnings.warn(
'Data for functions from both the noisy and ' +
'non-noisy testbeds have been found. Their ' +
'results will be mixed in the "all functions" ' +
'ECDF figures.')
algorithm_name0 = str_to_latex(strip_pathname1(sortedAlgs[0]))
algorithm_name1 = str_to_latex(strip_pathname1(sortedAlgs[1]))
algorithm_name = "%s vs %s" % (algorithm_name1,
algorithm_name0)
ppfig.save_single_functions_html(
os.path.join(many_algorithms_output,
genericsettings.pprldistr2_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPRLDISTR2,
function_groups=ds_list0.getFuncGroups(),
parentFileName=genericsettings.many_algorithm_file_name)
# ECDFs of aRT ratios
dic_dim0 = ds_list0.dictByDim()
dic_dim1 = ds_list1.dictByDim()
for dim in set(dic_dim0.keys()) & set(dic_dim1.keys()):
if dim in testbedsettings.current_testbed.rldDimsOfInterest:
# ECDF for all functions altogether
try:
pprldistr2.main(
dic_dim0[dim], dic_dim1[dim], dim,
testbedsettings.current_testbed.
rldValsOfInterest, many_algorithms_output,
'%02dD_all' % dim)
except KeyError:
warnings.warn('Could not find some data in %d-D.' %
dim)
continue
# ECDFs per function groups
dict_fun_group0 = dic_dim0[dim].dictByFuncGroup()
dict_fun_group1 = dic_dim1[dim].dictByFuncGroup()
for fGroup in set(dict_fun_group0.keys()) & set(
dict_fun_group1.keys()):
pprldistr2.main(
dict_fun_group1[fGroup],
dict_fun_group0[fGroup], dim, testbedsettings.
current_testbed.rldValsOfInterest,
many_algorithms_output,
'%02dD_%s' % (dim, fGroup))
# ECDFs per noise groups
dict_fun0 = dic_dim0[dim].dictByNoise()
dict_fun1 = dic_dim1[dim].dictByNoise()
for fGroup in set(dict_fun0.keys()) & set(
dict_fun1.keys()):
pprldistr2.main(
dict_fun1[fGroup], dict_fun0[fGroup], dim,
testbedsettings.current_testbed.
rldValsOfInterest, many_algorithms_output,
'%02dD_%s' % (dim, fGroup))
prepend_to_file(
latex_commands_file,
[
'\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
pprldistr.caption_two(), # depends on the config
# setting, should depend
# on maxfevals
'}'
])
print_done()
if testbedsettings.current_testbed not in [
testbedsettings.GECCOBiObjBBOBTestbed,
testbedsettings.GECCOBiObjExtBBOBTestbed
]:
print("ECDF runlength graphs...")
for dim in set(dic_dim0.keys()) & set(dic_dim1.keys()):
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
# ECDFs of all functions altogether
if dim in testbedsettings.current_testbed.rldDimsOfInterest:
try:
pprldistr.comp(
dic_dim1[dim],
dic_dim0[dim],
testbedsettings.current_testbed.
rldValsOfInterest,
# TODO: let rldVals... possibly be RL-based targets
True,
many_algorithms_output,
'all')
except KeyError:
warnings.warn(
'Could not find some data in %d-D.' % dim)
continue
# ECDFs per function groups
dict_fun_group0 = dic_dim0[dim].dictByFuncGroup()
dict_fun_group1 = dic_dim1[dim].dictByFuncGroup()
for fGroup in set(dict_fun_group0.keys()) & set(
dict_fun_group1.keys()):
pprldistr.comp(
dict_fun_group1[fGroup],
dict_fun_group0[fGroup], testbedsettings.
current_testbed.rldValsOfInterest, True,
many_algorithms_output, '%s' % fGroup)
# ECDFs per noise groups
dict_fun0 = dic_dim0[dim].dictByNoise()
dict_fun1 = dic_dim1[dim].dictByNoise()
for fGroup in set(dict_fun0.keys()) & set(
dict_fun1.keys()):
pprldistr.comp(
dict_fun1[fGroup], dict_fun0[fGroup],
testbedsettings.current_testbed.
rldValsOfInterest, True,
many_algorithms_output, '%s' % fGroup)
print_done() # of "ECDF runlength graphs..."
# ECDFs per noise groups
print("ECDF graphs per noise group...")
grouped_ecdf_graphs(pproc.dictAlgByNoi(dictAlg), sortedAlgs,
many_algorithms_output,
dictAlg[sortedAlgs[0]].getFuncGroups(), inset,
genericsettings.many_algorithm_file_name)
print_done()
# ECDFs per function groups
print("ECDF graphs per function group...")
grouped_ecdf_graphs(pproc.dictAlgByFuncGroup(dictAlg), sortedAlgs,
many_algorithms_output,
dictAlg[sortedAlgs[0]].getFuncGroups(), inset,
genericsettings.many_algorithm_file_name)
print_done()
# copy-paste from above, here for each function instead of function groups:
print("ECDF graphs per function...")
if genericsettings.isRldOnSingleFcts:
# ECDFs for each function
if 1 < 3:
pprldmany.all_single_functions(
dictAlg,
False,
sortedAlgs,
many_algorithms_output,
genericsettings.many_algorithm_file_name,
settings=inset)
else: # subject to removal
dictFG = pproc.dictAlgByFun(dictAlg)
for fg, tmpdictAlg in dictFG.items():
dictDim = pproc.dictAlgByDim(tmpdictAlg)
dims = sorted(dictDim)
for i, d in enumerate(dims):
entries = dictDim[d]
single_fct_output_dir = (
many_algorithms_output.rstrip(os.sep) +
os.sep + 'pprldmany-single-functions'
# + os.sep + ('f%03d' % fg)
)
if not os.path.exists(single_fct_output_dir):
os.makedirs(single_fct_output_dir)
pprldmany.main(entries,
order=sortedAlgs,
outputdir=single_fct_output_dir,
info=('f%03d_%02dD' % (fg, d)),
settings=inset)
ppfig.save_single_functions_html(
os.path.join(single_fct_output_dir,
genericsettings.pprldmany_file_name),
'', # algorithms names are clearly visible in the figure
dimensions=dims,
htmlPage=ppfig.HtmlPage.NON_SPECIFIED,
header=ppfig.pprldmany_per_func_dim_header)
print_done()
if prepare_tables:
print("Generating comparison tables...")
prepend_to_file(latex_commands_file, [
'\providecommand{\\bbobpptablesmanylegend}[1]{' +
pptables.get_table_caption() + '}'
])
dictNoi = pproc.dictAlgByNoi(dictAlg)
for ng, tmpdictng in dictNoi.items():
dictDim = pproc.dictAlgByDim(tmpdictng)
for d, tmpdictdim in sorted(dictDim.items()):
pptables.main(
tmpdictdim, sortedAlgs, many_algorithms_output,
([1, 20, 38] if
(testbedsettings.current_testbed.name
== testbedsettings.testbed_name_bi) else True),
latex_commands_file)
print_done()
if prepare_scatter and len(
genericsettings.foreground_algorithm_list) == 2:
print("Scatter plots...")
ds_list0 = dictAlg[sortedAlgs[0]]
algorithm_name0 = str_to_latex(strip_pathname1(sortedAlgs[0]))
ds_list1 = dictAlg[sortedAlgs[1]]
algorithm_name1 = str_to_latex(strip_pathname1(sortedAlgs[1]))
algorithm_name = "%s vs %s" % (algorithm_name1, algorithm_name0)
ppfig.save_single_functions_html(
os.path.join(many_algorithms_output,
genericsettings.ppscatter_file_name),
algname=algorithm_name,
htmlPage=ppfig.HtmlPage.PPSCATTER,
function_groups=ds_list0.getFuncGroups(),
parentFileName=genericsettings.many_algorithm_file_name)
html_file_name = os.path.join(
many_algorithms_output,
genericsettings.ppscatter_file_name + '.html')
ppscatter.main(ds_list1, ds_list0, many_algorithms_output, inset)
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobppscatterlegend}[1]{',
ppscatter.figure_caption(), '}'
])
replace_in_file(html_file_name, '##bbobppscatterlegend##',
ppscatter.figure_caption(for_html=True))
for i, alg in enumerate(args):
replace_in_file(html_file_name,
'algorithm' + pptex.numtotext(i),
str_to_latex(strip_pathname1(alg)))
print_done()
if prepare_figures:
print("Scaling figures...")
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
plt.rc('pdf', fonttype=42)
ppfigs.main(dictAlg, genericsettings.ppfigs_file_name, sortedAlgs,
many_algorithms_output, latex_commands_file)
plt.rcdefaults()
print_done()
print("Output data written to folder %s" %
os.path.join(os.getcwd(), many_algorithms_output))
plt.rcdefaults()
return DataSetList(dsList).dictByAlg()
def plot_sobol(data, filename):
"""
Plot the total-order Sobol indices from the uncertainty quantification.
Parameters
----------
data : uncertainpy.Data object
The data from the uncertainty quantification.
filename : str
Filename of the figure.
"""
feature_titles = {
"spike_rate": "Event rate",
"average_AP_overshoot": "Average event peak",
"average_AHP_depth": "Average AHP depth",
"burstiness_factor": "Burstiness factor",
"average_duration": "Average duration"
}
nr_plots = len(features_to_run)
grid_size = np.ceil(np.sqrt(nr_plots))
grid_x_size = int(grid_size)
grid_y_size = int(np.ceil(nr_plots / float(grid_x_size)))
style = "seaborn-darkgrid"
set_style(style)
plt.rcParams.update({"axes.titlepad": 8, "font.family": "serif"})
fig, axes = plt.subplots(nrows=grid_y_size,
ncols=grid_x_size,
squeeze=False,
figsize=(figure_width, figure_width * 0.8))
# Add a larger subplot to use to set a common xlabel and ylabel
set_style("seaborn-white")
ax = fig.add_subplot(111, zorder=-10)
spines_color(ax,
edges={
"top": "None",
"bottom": "None",
"right": "None",
"left": "None"
})
ax.tick_params(top=False,
bottom=False,
left=False,
right=False,
labelsize=labelsize,
labelbottom=False,
labelleft=False)
ax.set_ylabel("Total-order Sobol indices",
labelpad=35,
fontsize=titlesize,
fontweight=fontweight)
width = 0.2
index = np.arange(1, len(data.uncertain_parameters) + 1) * width
latex_labels = {
"g_K": r"$g_\mathrm{K}$",
"g_Ca": r"$g_\mathrm{Ca}$",
"g_SK": r"$g_\mathrm{SK}$",
"g_Na": r"$g_\mathrm{Na}$",
"g_l": r"$g_\mathrm{l}$",
"g_BK": r"$g_\mathrm{BK}$"
}
xlabels = []
for label in data.uncertain_parameters:
xlabels.append(latex_labels[label])
for i in range(0, grid_x_size * grid_y_size):
nx = i % grid_x_size
ny = int(np.floor(i / float(grid_x_size)))
ax = axes[ny][nx]
if i < nr_plots:
title = feature_titles[features_to_run[i]]
sensitivity = data[features_to_run[i]].sobol_total_average
mean = data[features_to_run[i]].mean
std = np.sqrt(data[features_to_run[i]].variance)
unit = data[features_to_run[i]].labels
if unit and "(" in unit[0]:
unit = unit[0].split("(")[-1].strip(")")
else:
unit = ""
# Convert spike rate from 1/ms to Hz (1/s)
if features_to_run[i] == "spike_rate":
mean *= 1000
std *= 1000
unit = "Hz"
prettyBar(sensitivity,
xlabels=xlabels,
nr_colors=len(data.uncertain_parameters),
index=index,
palette="colorblind",
ax=ax,
style=style)
for tick in ax.get_xticklabels():
tick.set_rotation(-40)
ax.set_ylim([0, 1.15])
ax.set_title(title, fontsize=titlesize)
ax.text(label_x,
label_y,
string.ascii_uppercase[i],
transform=ax.transAxes,
fontsize=titlesize,
fontweight=plot_label_weight)
ax.text(0.1,
0.9,
"Mean = {mean:.2{c}} {unit}".format(
mean=mean,
c="e" if abs(mean) < 1e-2 else "f",
unit=unit),
transform=ax.transAxes,
fontsize=labelsize,
fontweight=fontweight)
ax.text(0.1,
0.8,
"Std. = {std:.2{c}} {unit}".format(
std=std, c="e" if abs(mean) < 1e-2 else "f",
unit=unit),
transform=ax.transAxes,
fontsize=labelsize,
fontweight=fontweight)
else:
ax.axis("off")
plt.tight_layout()
plt.subplots_adjust(bottom=0.13, left=0.1)
plt.savefig(filename + figure_format)
# Reset matplotlib defaults
plt.rcdefaults()
def NNTRegressor(I_data_path, I_Model_path, k, num_epochs, batch_Size, Verbose, NNTList):
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import time
import os
import warnings
warnings.filterwarnings('ignore')
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_absolute_error
import keras
from keras.models import Sequential
from keras import models
from keras import layers
get_ipython().run_line_magic('matplotlib', 'inline')
from joblib import dump
#################################### Functions ####################################
# Normalise
def Normalise(Data, Target):
sc = StandardScaler()
dfX = Data.drop([Target], axis=1)
sc.fit(dfX)
X = sc.transform(dfX)
os.makedirs(I_Model_path + 'temp', exist_ok=True)
print('save scaler to '+ I_Model_path + 'temp/scaler.pickle')
with open(I_Model_path + 'temp/scaler.pickle', 'wb') as f:
pickle.dump(sc, f)
dump(sc, I_Model_path + 'temp/std_scaler.bin', compress=True)
y = Data.values[:, Data.columns.get_loc(Target)]
return X, y
#Validation
def validation(train_data, train_targets, num_val_samples, i):
val_data = train_data[i * num_val_samples: (i + 1) * num_val_samples]
val_targets = train_targets[i * num_val_samples: (i + 1) * num_val_samples]
return val_data, val_targets
def partial_train(train_data, train_targets, num_val_samples, i):
partial_train_data = np.concatenate([train_data[:i * num_val_samples], train_data[(i + 1) * num_val_samples:]],
axis=0)
partial_train_targets = np.concatenate([train_targets[:i * num_val_samples], train_targets[(i + 1) * num_val_samples:]],
axis=0)
return partial_train_data, partial_train_targets
#Models
def build_model1(titles):
red = Sequential()
red.add(layers.Dense(64, activation='relu', input_shape=(n,)))
red.add(layers.Dense(128, activation='relu'))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'Three hidden layers (relu: 64, 128, 64) and rmsprop optimizer'
titles.append(title)
return red
def build_model2(titles):
red = Sequential()
red.add(layers.Dense(16, activation='relu', input_shape=(n,)))
red.add(layers.Dense(16, activation='relu'))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'Two hidden layers (relu: 16) and rmsprop optimizer'
titles.append(title)
return red
def build_model3(titles):
red = Sequential()
red.add(layers.Dense(512, activation='relu', input_shape=(n,)))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'One hidden layers (relu: 512) and rmsprop optimizer'
titles.append(title)
return red
def build_model4(title):
red = Sequential()
red.add(layers.Dense(64, activation='relu', input_shape=(n,)))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(64, activation='softmax'))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'Three hidden layers (relu: 64) and rmsprop optimizer'
titles.append(title)
return red
def build_model5(title):
red = Sequential()
red.add(layers.Dense(32, activation='relu', input_shape=(n,)))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(128, activation='relu'))
red.add(layers.Dense(128, activation='relu'))
red.add(layers.Dense(512, activation='relu'))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'One hidden layers (sigmoid: 16) and rmsprop optimizer'
titles.append(title)
return red
def build_model6(titles):
red = Sequential()
red.add(layers.Dense(64, activation='relu', input_shape=(train_data.shape[1],)))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(64, activation='relu'))
red.add(layers.Dense(1))
red.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
title = 'Four hidden layers (relu: 64) and rmsprop optimizer'
titles.append(title)
return red
#MAE in k
def average_mae(train_all_scores, val_all_scores, num_epochs):
training = [np.mean([x[k] for x in train_all_scores]) for k in range(num_epochs)]
validation = [np.mean([x[k] for x in val_all_scores]) for k in range(num_epochs)]
return training, validation
#Visuals
def plot_Mae(average_mae_training, average_mae_validation, title, i):
my_dpi=96
plt.figure(figsize=(1000/my_dpi, 1000/my_dpi), dpi=my_dpi)
plt.plot(range(1, len(average_mae_training) + 1), average_mae_training, 'o', label='Training')
plt.plot(range(1, len(average_mae_validation) + 1), average_mae_validation, 'lightseagreen', label='Validation')
plt.title(title, size=18)
plt.xlabel('Epochs', size=18)
plt.ylabel('Mean Absolute Error', size=18)
plt.legend(loc='best',prop={'size': 16})
os.makedirs(I_Model_path + 'visuals', exist_ok=True)
plt.savefig(I_Model_path + 'visuals/MAE Model' + str(NNTList[i]) + '.png')
# Timer
def Timer(Model_Step, last_time, k):
Time_list = []
now = time.time() - last_time
if now < 60:
print(' Time of '+ Model_Step+str(k)+': '+ str(round(now))+'s')
Time_list.append('Time of '+ Model_Step +str(k)+': '+ str(round(now))+'s')
elif 60<now<3600:
now_m = now/60
minutes = int(now_m)
secs = int((now_m - minutes)*60)
print(' Time of '+ Model_Step+str(k)+': ' +str(minutes)+'m and '+str(secs)+'s')
Time_list.append(' Time of '+ Model_Step+str(k)+': ' +str(minutes)+'m and '+str(secs)+'s')
del now_m
del minutes
del secs
elif now>3600:
now_h = now/3600
hours = int(now_h)
now_m = (now_h - hours)*60
minutes = int(now_m)
secs = int((now_m - minutes)*60)
print(' Time of '+ Model_Step+str(k)+': ' +str(hours)+'h, '+str(minutes)+'m and '+str(secs)+'s')
Time_list.append(' Time of '+ Model_Step+str(k)+': ' +str(hours)+'h, '+str(minutes)+'m and '+str(secs)+'s')
del now_h
del hours
del now_m
del minutes
del secs
############################################################################################################
# Define inputs
Master_path = I_data_path
os.makedirs(I_Model_path, exist_ok=True)
## Load data:
print('Loading the data...')
data = pd.read_csv(Master_path)
#Creating a dataset copy
Data = data.copy()
print('Working dataset has: ' + str(Data.shape[0])+ ' rows and ' + str(Data.shape[1])+ ' columns')
# Normalise
print('Normalising...')
X, y = Normalise(Data,'CANTIDADSALIDA')
print('Completed')
#Split in train test
print('Spliting in train and test set')
train_data, test_data, train_targets, test_targets = train_test_split(X, y,
train_size=0.80, test_size=0.20, random_state=0)
### Save as csv:
os.makedirs(I_Model_path + 'temp', exist_ok=True)
dfX = Data.drop('CANTIDADSALIDA', axis=1)
print('Saving training feature set to ' + I_Model_path + 'temp/train_X.csv')
train_data_df = pd.DataFrame(train_data, columns= dfX.columns )
train_data_df.to_csv(I_Model_path + 'temp/train_X.csv')
with open(I_Model_path + 'temp/train_X.pickle', 'wb') as f:
pickle.dump(train_data, f)
print('Saving training labels to '+I_Model_path + 'temp/train_Y.csv')
train_targets_df = pd.DataFrame(train_targets, columns= ['CANTIDADSALIDA'])
train_targets_df.to_csv(I_Model_path + 'temp/train_Y.csv')
with open(I_Model_path + 'temp/train_Y.pickle', 'wb') as f:
pickle.dump(train_targets, f)
print('Saving test feature set to '+ I_Model_path + 'temp/test_X.csv')
test_data_df = pd.DataFrame(test_data, columns= dfX.columns )
test_data_df.to_csv(I_Model_path + 'temp/test_X.csv')
with open(I_Model_path + 'temp/test_X.pickle', 'wb') as f:
pickle.dump(test_data, f)
print('Saving test labels to '+I_Model_path + '/temp/test_Y.csv')
test_targets_df = pd.DataFrame(test_targets, columns= ['CANTIDADSALIDA'])
test_targets_df.to_csv(I_Model_path + 'temp/test_Y.csv')
with open(I_Model_path + 'temp/test_Y.pickle', 'wb') as f:
pickle.dump(test_targets, f)
train_data_df["CANTIDADSALIDA"] = train_targets_df["CANTIDADSALIDA"]
train_data_df['CANTIDADSALIDA'] = train_data_df['CANTIDADSALIDA'].astype(int)
test_data_df["CANTIDADSALIDA"] = test_targets_df["CANTIDADSALIDA"]
test_data_df['CANTIDADSALIDA'] = test_data_df['CANTIDADSALIDA'].astype(int)
# Parameters
#A) Lists:
models = []
titles = []
#B) Temps
n = train_data.shape[1] # number of features
num_val_samples = len(train_data) // k
#Model
if 1 in NNTList:
models.append(build_model1(titles))
if 2 in NNTList:
models.append(build_model2(titles))
if 3 in NNTList:
models.append(build_model3(titles))
if 4 in NNTList:
models.append(build_model4(titles))
if 5 in NNTList:
models.append(build_model5(titles))
if 6 in NNTList:
models.append(build_model6(titles))
# Training K-Cross Neural Network:
print('- Starting to train the model:')
final_time = time.time()
for i in range(len(models)):
last_time = time.time()
print('Model '+ str(NNTList[i]) + ':')
# NNT model
red = models[i]
# Lists
train_all_scores_byModel = []
val_all_scores_byModel = []
#K-Cross-validation
for j in range(k):
last_time2 = time.time()
print(" Processing fold #", j)
# Validation
val_data, val_targets = validation(train_data, train_targets, num_val_samples, j)
partial_train_data, partial_train_targets = partial_train(train_data, train_targets, num_val_samples, j)
# Training
training = red.fit(partial_train_data, partial_train_targets, validation_data = (val_data, val_targets),
epochs = num_epochs, batch_size = batch_Size, verbose=Verbose)
#Mean absolute error
train_mae_training = training.history['mae']
val_mae_training = training.history['val_mae']
# Adding to the list
train_all_scores_byModel.append(train_mae_training)
val_all_scores_byModel.append(val_mae_training)
#timer
Timer('Step ', last_time2, j)
del val_data, val_targets
#Saving the model
print('Saving temps')
os.makedirs(I_Model_path + 'models', exist_ok=True)
os.makedirs(I_Model_path + 'temp', exist_ok=True)
red.save(I_Model_path + 'models/Model' +str(NNTList[i])+ '.h5')
with open(I_Model_path + 'models/NNTModel' +str(NNTList[i])+ '.pickle', 'wb') as f:
pickle.dump(red, f)
#timer
print(' ' + str(Timer('Model ', last_time, NNTList[i])))
# Evaluate
test_mse_score, test_mae_score = red.evaluate(test_data, test_targets,verbose=0)
print(" Mean Absolute Error model "+ str(NNTList[i])+ ":" "= {:.2f}".format(test_mae_score))
average_mae_training, average_mae_validation = average_mae(train_all_scores_byModel, val_all_scores_byModel, num_epochs)
with open(I_Model_path + 'temp/NNT_error' +str(NNTList[i])+ '.pickle', 'wb') as f:
pickle.dump(test_mae_score, f)
# Visual
plot_Mae(average_mae_training, average_mae_validation, titles[i], i)
plt.rcdefaults()
print(' ' + str(Timer('Global', final_time, 0)))
return train_data_df, train_targets_df, test_data_df, test_targets_df
def pca_arctilt(slf, tiltang, centval, tilts, maxp=25, maskval=-999999.9):
""" Generate a QA plot for the blaze function fit to each slit
Parameters
----------
slf : class
Science Exposure class
tiltang : ndarray
(m x n) 2D array containing the measured tilts (m) for each slit (n)
centval : ndarray
(m x n) 2D array containing the pixel location (in the spectral direction) of the measured tilts (m) for each slit (n)
tilts : ndarray
(m x n) 2D array containing the model tilts (m) for each slit (n)
maxp : int, (optional)
Maximum number of panels per page
maskval : float, (optional)
Value used in arrays to indicate a masked value
"""
plt.rcdefaults()
plt.rcParams['font.family'] = 'times new roman'
method = inspect.stack()[0][3]
outroot = arqa.set_qa_filename(slf.setup, method)
#
npc = tiltang.shape[1]
pages, npp = arqa.get_dimen(npc, maxp=maxp)
x0 = np.arange(tilts.shape[0])
# First calculate the min and max values for the plotting axes, to make sure they are all the same
w = np.where(tiltang != maskval)
medv = np.median(tiltang[w])
madv = 1.4826 * np.median(np.abs(medv - tiltang[w]))
ymin, ymax = medv - 3.0 * madv, medv + 3.0 * madv
ymin = min(ymin, np.min(tilts))
ymax = max(ymax, np.max(tilts))
# Check that ymin and ymax are set, if not, return without plotting
if ymin is None or ymax is None:
msgs.warn("Arc tilt fits were not plotted")
return
# Generate the plots
ndone = 0
for i in range(len(pages)):
f, axes = plt.subplots(pages[i][1], pages[i][0])
ipx, ipy = 0, 0
for j in range(npp[i]):
if pages[i][1] == 1:
ind = (ipx, )
elif pages[i][0] == 1:
ind = (ipy, )
else:
ind = (ipy, ipx)
w = np.where(tiltang[:, ndone] != maskval)
if np.size(w[0]) != 0:
axes[ind].plot(centval[:, ndone][w], tiltang[:, ndone][w],
'bx')
axes[ind].plot(x0, tilts[:, ndone], 'r-')
axes[ind].axis([0, tilts.shape[0] - 1, ymin, ymax])
axes[ind].set_title("Slit {0:d}".format(1 + ndone))
axes[ind].tick_params(labelsize=8)
ipx += 1
if ipx == pages[i][0]:
ipx = 0
ipy += 1
ndone += 1
# Delete the unnecessary axes
for j in range(npp[i], axes.size):
if pages[i][1] == 1:
ind = (ipx, )
elif pages[i][0] == 1:
ind = (ipy, )
else:
ind = (ipy, ipx)
f.delaxes(axes[ind])
ipx += 1
if ipx == pages[i][0]:
ipx = 0
ipy += 1
# Save the figure
if pages[i][1] == 1 or pages[i][0] == 1:
ypngsiz = 11.0 / axes.size
else:
ypngsiz = 11.0 * axes.shape[0] / axes.shape[1]
f.set_size_inches(11.0, ypngsiz)
f.tight_layout()
outfile = outroot + '{:02d}.png'.format(i)
plt.savefig(outfile, dpi=200)
plt.close()
#f.clf()
#del f
plt.rcdefaults()
return
def bar_charts(self):
start_time = time.time()
plt.rcdefaults()
objects = ('Good', 'Bad', 'Neutral')
x = np.arange(len(objects))
height = [
self.mark_counts_1rule[0], self.mark_counts_1rule[1],
self.mark_counts_1rule[2]
]
plt.bar(x, height, align='center', alpha=0.5)
plt.xticks(x, objects)
plt.ylabel('Count')
plt.title('Classification by sum')
plt.savefig(r'E:\data\1 bar chart.png')
height = [
self.mark_counts_2rule[0], self.mark_counts_2rule[1],
self.mark_counts_2rule[2]
]
plt.bar(x, height, align='center', alpha=0.5)
plt.xticks(x, objects)
plt.ylabel('Count')
plt.title('Classification by dominance')
plt.savefig(r'E:\data\2 bar chart.png')
height = [
self.mark_counts_3rule[0], self.mark_counts_3rule[1],
self.mark_counts_3rule[2]
]
plt.bar(x, height, align='center', alpha=0.5)
plt.xticks(x, objects)
plt.ylabel('Count')
plt.title('Classification by speech part diversification')
plt.savefig(r'E:\data\3 bar chart.png')
height = [
self.mark_counts_4rule[0], self.mark_counts_4rule[1],
self.mark_counts_4rule[2]
]
plt.bar(x, height, align='center', alpha=0.5)
plt.xticks(x, objects)
plt.ylabel('Count')
plt.title('Classification by average mark')
plt.savefig(r'E:\data\4 bar chart.png')
plt.show()
x = ('1st', '2nd', '3d', '4th', '5th')
x_pos = np.arange(len(x))
# Толщина столбца
width = 0.2
# Значения по Oy
y_positive = [x for x in self.top_adj_pos.values()]
y_negative = [x for x in self.top_adj_neg.values()]
# Графики
plt.bar(x_pos, y_positive, width=width, label='Positive', color='y')
plt.bar(x_pos + width,
y_negative,
width=width,
label='Negative',
color='g')
plt.ylabel('Amount')
plt.xticks(x_pos + width / 2, x)
plt.title('Top-5 Positive/Negative Adjectives')
plt.legend()
plt.savefig(r'E:\data\Adjectives bar chart.png')
# Создаем области
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
# Значения по Ox
x = [key for key in self._time_distribution1]
x_pos = np.arange(len(x))
# Первый график
ax1.plot(x_pos,
self.pos_distribution1,
'g.-',
linewidth=1,
markersize=7,
label='N_pos')
ax1.plot(x_pos,
self.neg_distribution1,
'b.--',
linewidth=1,
markersize=7,
label='N_neg')
ax1.plot(x_pos,
self.neu_distribution1,
'r.-.',
linewidth=1,
markersize=7,
label='N_neu')
ax1.set_xticks([])
ax1.grid(True)
ax1.legend()
ax1.set_ylabel('Fraction')
# столбцы
ax2.stem(x_pos, self.current_twits_sum1, basefmt=" ")
# координатн ось
ax2.set_xticks(x_pos)
ax2.set_xticklabels(x)
ax2.grid(True)
ax2.set_ylabel('Number of tweets')
fig.suptitle('Distribution of tweets classes in time rule 1',
fontsize=16,
y=1.0)
fig.savefig(r'E:\data\Time distribution1.png')
"""2 правило """
# Создаем области
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
# Значения по Ox
x = [key for key in self._time_distribution2]
x_pos = np.arange(len(x))
# Первый график
ax1.plot(x_pos,
self.pos_distribution2,
'g.-',
linewidth=1,
markersize=7,
label='N_pos')
ax1.plot(x_pos,
self.neg_distribution2,
'b.--',
linewidth=1,
markersize=7,
label='N_neg')
ax1.plot(x_pos,
self.neu_distribution2,
'r.-.',
linewidth=1,
markersize=7,
label='N_neu')
ax1.set_xticks([])
ax1.grid(True)
ax1.legend()
ax1.set_ylabel('Fraction')
# столбцы
ax2.stem(x_pos, self.current_twits_sum2, basefmt=" ")
# координатн ось
ax2.set_xticks(x_pos)
ax2.set_xticklabels(x)
ax2.grid(True)
ax2.set_ylabel('Number of tweets')
fig.suptitle('Distribution of tweets classes in time rule 2',
fontsize=16,
y=1.0)
fig.savefig(r'E:\data\Time distribution2.png')
"""3 правило """
# Создаем области
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
# Значения по Ox
x = [key for key in self._time_distribution3]
x_pos = np.arange(len(x))
# Первый график
ax1.plot(x_pos,
self.pos_distribution3,
'g.-',
linewidth=1,
markersize=7,
label='N_pos')
ax1.plot(x_pos,
self.neg_distribution3,
'b.--',
linewidth=1,
markersize=7,
label='N_neg')
ax1.plot(x_pos,
self.neu_distribution3,
'r.-.',
linewidth=1,
markersize=7,
label='N_neu')
ax1.set_xticks([])
ax1.grid(True)
ax1.legend()
ax1.set_ylabel('Fraction')
# столбцы
ax2.stem(x_pos, self.current_twits_sum3, basefmt=" ")
# координатн ось
ax2.set_xticks(x_pos)
ax2.set_xticklabels(x)
ax2.grid(True)
ax2.set_ylabel('Number of tweets')
fig.suptitle('Distribution of tweets classes in time rule 3',
fontsize=16,
y=1.0)
fig.savefig(r'E:\data\Time distribution3.png')
"""4 правило """
# Создаем области
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(27, 6))
# Значения по Ox
x = [key for key in self._time_distribution4]
x_pos = np.arange(len(x))
# Первый график
ax1.plot(x_pos,
self.pos_distribution4,
'g.-',
linewidth=1,
markersize=7,
label='N_pos')
ax1.plot(x_pos,
self.neg_distribution4,
'b.--',
linewidth=1,
markersize=7,
label='N_neg')
ax1.plot(x_pos,
self.neu_distribution4,
'r.-.',
linewidth=1,
markersize=7,
label='N_neu')
ax1.set_xticks([])
ax1.grid(True)
ax1.legend()
ax1.set_ylabel('Fraction')
# столбцы
ax2.stem(x_pos, self.current_twits_sum4, basefmt=" ")
# координатн ось
ax2.set_xticks(x_pos)
ax2.set_xticklabels(x)
ax2.grid(True)
ax2.set_ylabel('Number of tweets')
fig.suptitle('Distribution of tweets classes in time rule 4',
fontsize=16,
y=1.0)
fig.savefig(r'E:\data\Time distribution4.png')
print("\nГрафики строятся за ",
time.time() - start_time, " секунд.\n")
from matplotlib import pyplot as pp
import numpy as np
ejey = ["Heap", "Counting", "Selection", "Bubble", "Insertion", "Quick", "Merge", "Shell", "Gnome", "Cocktail"]
ejex = [9.932100e-4, 2.773860e-3, 3.420000e-3, 2.098000e-5, 4.090000e-5, 2.870356e-2, 1.408150e-3, 2.671700e-4, 1.960000e-5, 2.575000e-5]
pp.rcdefaults()
fig, ax = pp.subplots()
y_pos = np.arange(len(ejey))
ax.barh(y_pos, ejex, color = ("brown", "red", "darkorange", "gold", "yellowgreen", "green", "lightseagreen", "mediumblue", "violet", "pink"))
ax.set_yticks(y_pos)
ax.set_yticklabels(ejey)
ax.set_ylabel("Ordenamientos")
ax.set_xlabel("Tiempo Promedio")
ax.set_title("Vectores Ordenados Tamaño 1000")
pp.show()
def predict(self):
train = self.train
# get input data
# print('self.councilArea',self.councilArea)
R_n = train.Regionname[train.CouncilArea == self.councilArea]
# print(R_n.iat[0])
# print(type(train.Regionname[train.CouncilArea == self.councilArea]))
self.regionName = R_n.iat[0]
cnt = train.Propertycount[train.CouncilArea == self.councilArea]
self.propertyCount = cnt.iat[0]
self.column = [
'Rooms', 'Type', 'Method', 'SellerG', 'Sold_Year', 'Distance',
'Bathroom', 'Car', 'Landsize', 'CouncilArea', 'Regionname',
'Propertycount'
]
self.inputData = [
self.room, self.type, self.method, self.sellerG, self.soldYear,
self.distance, self.bathroom, self.car, self.landSize,
self.councilArea, self.regionName, self.propertyCount
]
# print('Input data ', self.inputData)
self.input = pd.DataFrame(data=[self.inputData], columns=self.column)
# print('Property information:')
# print(self.input)
self.dummy_test = self.dummy_data(self.input, train)
self.dummy_test = self.dummy_test.drop(['Unnamed: 0'], axis=1)
# print('-----------')
# print(self.dummy_test)
# print('-----------')
# transfer the input to xgb data structure
dtest = xgb.DMatrix(data=self.dummy_test)
# save some regarding charts
if os.path.exists('images'):
os.chdir('./images')
else:
os.mkdir('images')
os.chdir('./images')
dictionary = {
'h': 'House',
'u': 'Unit',
't': 'Townhouse',
'dev site': 'Development Site',
'o res': 'Other Residential'
}
# chart 1:
name = f'Average Price Of {dictionary[self.env[1]]} In Different Regions'
if not os.path.exists(name + '.png'):
CSV1 = train[['Regionname', 'Landsize', 'Price', 'Type']]
CSV1 = CSV1[CSV1['Type'] == self.type]
CSV1 = CSV1.groupby('Regionname').sum()
CSV1['Avg'] = CSV1.apply(lambda x: x.Price / x.Landsize, axis=1)
CSV1 = CSV1['Avg']
try:
CSV1.plot.bar()
plt.tight_layout()
plt.ylabel('Unit: $/m²')
plt.title(name)
plt.savefig(name + '.png')
except:
pass
# chart 2: the average pricein different council area
name = f'Average Price Of {dictionary[self.type]} In Different Council Area'
if not os.path.exists(name + '.png'):
plt.rcdefaults()
CSV3 = train[['CouncilArea', 'Landsize', 'Price', 'Type']]
CSV3 = CSV3[CSV3['Type'] == self.type]
CSV3 = CSV3.groupby('CouncilArea').sum()
CSV3['Avg'] = CSV3.apply(lambda x: x.Price / x.Landsize, axis=1)
CSV3 = CSV3['Avg']
try:
CSV3.plot.bar()
plt.tight_layout()
plt.ylabel('Unit: $/m²')
plt.title(name)
plt.savefig(name + '.png')
except:
pass
# chart 3: the average price of council area in different year
name = f'Average Price Of {dictionary[self.type]} Over Years'
if not os.path.exists(name + '.png'):
plt.rcdefaults()
CSV3 = train[['Sold_Year', 'Landsize', 'Price', 'Type']]
CSV3 = CSV3[CSV3['Type'] == self.type]
CSV3 = CSV3.groupby('Sold_Year').sum()
CSV3['Avg'] = CSV3.apply(lambda x: x.Price / x.Landsize, axis=1)
CSV3 = CSV3['Avg']
try:
CSV3.plot.bar()
plt.tight_layout()
plt.ylabel('Unit: $/m²')
plt.title(name)
plt.savefig(name + '.png')
except:
pass
os.chdir('..')
# Prediction
if os.path.exists('ppp.model'):
print('Pre-trained model found, begin to predict price ...')
bst2 = xgb.Booster(model_file='ppp.model')
preds = bst2.predict(dtest)
print('Property information:')
print(self.input)
return preds
else:
print('Not found pre-trained model, we are training model now ...')
# train first then predict.
self.training(train=train)
bst2 = xgb.Booster(model_file='ppp.model')
preds = bst2.predict(dtest)
print('Property information:')
print(self.input)
return preds
def plot_pdf(av_image, limits=None, savedir='./', filename=None, show=True,
scale=(0,0), n_bins=50, fit_gaussian=True, returnimage=False, title=''):
''' Plots a probability distribution function of an image.
'''
# Import external modules
import numpy as np
import math
import pyfits as pf
import matplotlib.pyplot as plt
import matplotlib
from scipy.optimize import curve_fit
# Drop the NaNs from the images
indices = np.where(av_image == av_image)
av_image_nonans = av_image[indices]
# Set up plot aesthetics
plt.clf()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
#color_cycle = [colormap(i) for i in np.linspace(0, 0.9, len(flux_list))]
fontScale = 12
params = {#'backend': .pdf',
'axes.labelsize': fontScale,
'axes.titlesize': fontScale,
'text.fontsize': fontScale,
'legend.fontsize': fontScale*3/4,
'xtick.labelsize': fontScale,
'ytick.labelsize': fontScale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': False,
'figure.figsize': (6, 6),
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
# Create figure
plt.clf()
fig = plt.figure()
ax = fig.add_subplot(111)
# Derive the histograms
bin_edges = np.logspace(-3, 3, num=n_bins, base=np.e)
n = np.zeros(n_bins - 1)
for i in xrange(n_bins - 1):
bin_count = len(av_image_nonans[(av_image_nonans > bin_edges[i]) & \
(av_image_nonans < bin_edges[i + 1])])
n[i] = bin_count
bin_centers = 0.5*(bin_edges[1:] + bin_edges[:-1])
n = np.append(n, 0)
bin_centers = np.append(bin_centers,
bin_centers[-1] + (bin_centers[-1] - bin_centers[-2]))
bin_centers = np.log(bin_centers)
ax.errorbar(
bin_centers,
n,
yerr = n**0.5,
marker = '.',
color = 'k',
drawstyle = 'steps-mid'
)
# Fit a gausssian to the distribution
if fit_gaussian:
def gauss(x, a, x0, sigma):
return a * np.exp(-(x - x0)**2 / (2 * sigma**2))
popt, pcov = curve_fit(gauss, bin_centers, n, p0=[200, 1, 2])
ax.plot(bin_centers,
gauss(bin_centers, *popt),
color = 'r')
try:
if scale[0] == 0:
x_scale = 'linear'
elif scale[0] == 1:
x_scale = 'log'
if scale[1] == 0:
y_scale = 'linear'
elif scale[1] == 1:
y_scale = 'log'
except IndexError('Scale must be tuple with 2 integer elements.'):
pass
ax.set_xscale(x_scale, nonposx = 'clip')
ax.set_yscale(y_scale, nonposy = 'clip')
if limits is not None:
ax.set_xlim(limits[0],limits[1])
ax.set_ylim(limits[2],limits[3])
# Adjust asthetics
ax.set_xlabel(r'ln(A$_{\rm V}$ (mag))',)
ax.set_ylabel(r'N',)
ax.set_title(title)
ax.grid(True)
if filename is not None:
plt.savefig(savedir + filename,bbox_inches='tight')
if show:
fig.show()
def plot_av(av_x,
av_y,
av_x_error=None,
av_y_error=None,
limits=None,
savedir='./',
filename=None,
show=True,
scale='linear',
returnimage=False,
hess_binsize=None,
title='',
plot_type='scatter',
color_scale='linear',
errorbar=None,
errorbar_pos=None):
# Import external modules
import numpy as np
import math
import pyfits as pf
import matplotlib.pyplot as plt
import matplotlib
from matplotlib import cm
from mpl_toolkits.axes_grid1 import ImageGrid
# Drop the NaNs from the images
indices = np.where((av_x == av_x) &\
(av_y == av_y) &\
(av_x > 0) &\
(av_y > 0))
try:
av_x_nonans = av_x[indices]
av_y_nonans = av_y[indices]
if type(av_y_error) is float:
av_y_error_nonans = sd_image_error * \
np.ones(av_y[indices].shape)
else:
av_y_error_nonans = sd_image_error[indices]
if type(av_x_error) is np.ndarray:
av_x_error_nonans = av_x_error[indices]
else:
av_x_error_nonans = av_x_error * \
np.ones(av_x[indices].shape)
except NameError:
no_errors = True
# Create figure
# Set up plot aesthetics
plt.clf()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
#color_cycle = [colormap(i) for i in np.linspace(0, 0.9, len(flux_list))]
fig_size = (4, 4)
font_scale = 10
params = { #'backend': .pdf',
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
'text.fontsize': font_scale,
'legend.fontsize': font_scale * 3 / 4,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': False,
'figure.figsize': fig_size,
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
# Create figure
plt.clf()
fig = plt.figure()
imagegrid = ImageGrid(
fig,
(1, 1, 1),
nrows_ncols=(1, 1),
ngrids=1,
axes_pad=0.25,
aspect=False,
label_mode='L',
share_all=True,
cbar_mode='single',
cbar_pad=0.1,
cbar_size=0.2,
)
ax = imagegrid[0]
if av_y_error is None:
if plot_type is 'hexbin':
if color_scale == 'linear':
image = ax.hexbin(av_x_nonans.ravel(),
av_y_nonans.ravel(),
mincnt=1,
yscale=scale,
xscale=scale,
cmap=cm.gist_stern)
for i, pos in enumerate(errorbar_pos):
error_bar = ax.errorbar(pos[0],
pos[1],
xerr=(errorbar[0]),
yerr=(errorbar[1]),
color='c',
linewidth=2.4,
capsize=2.4)
ax.set_xscale(scale, nonposx='clip')
ax.set_yscale(scale, nonposy='clip')
cb = ax.cax.colorbar(image, )
# Write label to colorbar
cb.set_label_text('Bin Counts', )
elif color_scale == 'log':
image = ax.hexbin(av_x_nonans.ravel(),
av_y_nonans.ravel(),
norm=matplotlib.colors.LogNorm(),
mincnt=1,
yscale=scale,
xscale=scale,
gridsize=(100, 200),
cmap=cm.gist_stern)
for i, pos in enumerate(errorbar_pos):
error_bar = ax.errorbar(pos[0],
pos[1],
xerr=(errorbar[0]),
yerr=(errorbar[1]),
color='c')
ax.set_xscale(scale, nonposx='clip')
ax.set_yscale(scale, nonposy='clip')
# Adjust color bar of density plot
#cb = image.colorbar(image)
#cb.set_label('Bin Counts')
elif plot_type is 'scatter':
image = ax.scatter(av_x_nonans.ravel(),
av_y_nonans.ravel(),
alpha=0.3,
color='k')
ax.set_xscale(scale)
ax.set_yscale(scale)
else:
image = ax.errorbar(av_x_nonans.ravel(),
av_y_nonans.ravel(),
xerr=(av_x_error_nonans.ravel()),
yerr=(av_y_error_nonans.ravel()),
alpha=0.3,
color='k',
marker='^',
ecolor='k',
linestyle='none',
markersize=2)
ax.set_xscale(scale)
ax.set_yscale(scale)
if limits is not None:
ax.set_xlim(limits[0], limits[1])
ax.set_ylim(limits[2], limits[3])
# Adjust asthetics
ax.set_xlabel(r'Planck A$_{\rm V}$ (mag)')
ax.set_ylabel(r'K+09 A$_{\rm V}$ (mag)')
ax.set_title(title)
ax.grid(True)
if filename is not None:
plt.savefig(savedir + filename, bbox_inches='tight', dpi=600)
if show:
fig.show()
if returnimage:
return correlations_image
def parameter_plotting(dataset, data_dir, plot_dir, pairwise=False):
plt.rcdefaults()
# IMO very sloppy way to do it
clear_name = lambda X: X.split(':')[-1] if X.split(':')[
0] == 'classifier' else X.split(':')[0]
#Styles
sns.set_style(
'whitegrid', {
'axes.linewidth': 1.25,
'axes.edgecolor': '0.15',
'grid.linewidth': 1.5,
'grid.color': 'gray'
})
sns.set_color_codes()
plt.rcParams['figure.figsize'] = (12.0, 9.0)
plt.rc('text', usetex=False)
plt.rc('font', size=13.0, family='serif')
preprocessor = 'NoPreprocessing'
## Parameter importance table
state_run_dir = os.path.join(data_dir, dataset, preprocessor,
'merged_runs')
# fanova_set = pyfanova.fanova.Fanova(state_run_dir)
fanova_set = pyfanova.fanova.Fanova(state_run_dir,
improvement_over='QUANTILE',
quantile_to_compare=0.25)
max_marginals = 7
cols_imp_df = ['marginal', 'parameter']
temp_df = pd.DataFrame(fanova_set.print_all_marginals(
max_num=max_marginals, pairwise=pairwise),
columns=cols_imp_df)
flatex = '%d_marginal_table_for_%s_over_q1_noprepro.tex' % (max_marginals,
dataset)
# flatex = '%d_marginal_table_for_%s_default_noprepro.tex' % (max_marginals, dataset)
# To avoid dots
pd.set_option('display.max_colwidth', -1)
temp_df.to_latex(os.path.join(plot_dir, 'tables', flatex))
print("Done printing latex")
pd.set_option('display.max_colwidth', 51)
if pairwise:
temp_df.loc[:, 'parameter'] = temp_df.parameter.str.split(' x ')
## Plot now the marginals
viz_set = pyfanova.visualizer.Visualizer(fanova_set)
categorical_params = fanova_set.get_config_space(
).get_categorical_parameters()
for p in temp_df.parameter:
fig_hyper, ax_hyper = plt.subplots(1, 1)
if len(p) == 1:
label = clear_name(p[0])
if p[0] not in categorical_params:
viz_set.plot_marginal(p[0], ax=ax_hyper)
else:
viz_set.plot_categorical_marginal(p[0], ax=ax_hyper)
ax_hyper.set_xlabel(label)
else:
label = clear_name(p[0]) + '_X_' + clear_name(p[1])
if p[0] in categorical_params:
if p[1] not in categorical_params:
viz_set.plot_categorical_pairwise(p[0], p[1], ax=ax_hyper)
ax_hyper.set_xlabel(clear_name(p[1]))
ax_hyper.legend(loc='best', title=clear_name(p[0]))
else:
continue
else:
if p[1] not in categorical_params:
viz_set.plot_contour_pairwise(p[0], p[1], ax=ax_hyper)
ax_hyper.set_xlabel(clear_name(p[0]))
ax_hyper.set_ylabel(clear_name(p[1]))
else:
viz_set.plot_categorical_pairwise(p[1], p[0], ax=ax_hyper)
ax_hyper.set_xlabel(clear_name(p[0]))
ax_hyper.legend(loc='best', title=clear_name(p[1]))
plt.tight_layout()
# fig_hyper.savefig(os.path.join(plot_dir, '%s_for_%s_noprepro.pdf' % (label, dataset)))
fig_hyper.savefig(
os.path.join(plot_dir,
'%s_for_%s_over_q1_noprepro.pdf' % (label, dataset)))
def plot_av_residuals(av_image1=None,
av_image2=None,
title=None,
limits=None,
savedir='./',
filename=None,
show=True,
header=None):
# Import external modules
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from mpl_toolkits.axes_grid1 import ImageGrid
import pyfits as pf
import matplotlib.pyplot as plt
import pywcsgrid2 as wcs
import pywcs
from pylab import cm # colormaps
from matplotlib.patches import Polygon
# Set up plot aesthetics
plt.clf()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
#color_cycle = [colormap(i) for i in np.linspace(0, 0.9, len(flux_list))]
font_scale = 12
params = { #'backend': .pdf',
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
'text.fontsize': font_scale,
'legend.fontsize': font_scale * 3 / 4,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': False,
'figure.figsize': (8, 6),
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
# Create figure instance
fig = plt.figure()
print(type(header))
imagegrid = ImageGrid(fig, (1, 1, 1),
nrows_ncols=(1, 1),
ngrids=1,
cbar_mode="single",
cbar_location='right',
cbar_pad="2%",
cbar_size='3%',
axes_pad=0,
axes_class=(wcs.Axes, dict(header=header)),
aspect=False,
label_mode='L',
share_all=False)
av_image1 = np.ma.array(av_image1, mask=(av_image1 != av_image1))
av_image2 = np.ma.array(av_image2, mask=(av_image2 != av_image2))
# create axes
ax = imagegrid[0]
cmap = cm.gist_stern # colormap
# show the image
im = ax.imshow(
av_image1 - av_image2,
interpolation='nearest',
origin='lower',
vmin=-9,
vmax=2,
#norm=matplotlib.colors.LogNorm(vmin=0.001, vmax=50,),
cmap=cmap,
)
# Asthetics
ax.set_display_coord_system("fk4")
ax.set_ticklabel_type("hms", "dms")
ax.set_xlabel('Right Ascension (J2000)', )
ax.set_ylabel('Declination (J2000)', )
if title is not None:
ax.set_title(title)
# colorbar
cb = ax.cax.colorbar(im, ) #ticks=[0.01, 0.1, 1, 10])
#cb.ax.set_xticklabels(['%.2f'%0.01, '0.1', '1', '10'])# horizontal colorbar
# plot limits
if limits is not None:
ax.set_xlim(limits[0], limits[2])
ax.set_ylim(limits[1], limits[3])
# Write label to colorbar
cb.set_label_text('Residuals (Mag)', )
if filename is not None:
plt.savefig(savedir + filename)
if show:
fig.show()
import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
def wordfreq(str):
counts = dict()
words = str.split()
for word in words:
if word in counts:
counts[word] += 1
else:
counts[word] = 1
return counts
text1 = []
filename1 = "pl.txt"
with open(filename1) as paradise:
for line in paradise:
replace = line.replace("\r", "").replace("\n", "").replace('"', '').replace(".", "").replace(",","").replace("?", "").replace(":", "").replace(";", "").replace("!", "").replace("(","").replace(")","").replace("--", "")
text1.append(replace.lower())
bigassbook = " ".join(text1)
print(bigassbook)
print(wordfreq(bigassbook))
god = wordfreq(bigassbook).get("god")
satan = wordfreq(bigassbook).get("satan")
def plot_av_images(cloud_dict, title=None, boxes=False, savedir='./',
filename=None, show=True, rh2_limits=None, hi_sd_limits=None):
# Import external modules
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from mpl_toolkits.axes_grid1 import ImageGrid
import pyfits as pf
import matplotlib.pyplot as plt
import pywcsgrid2
import pywcs
from pylab import cm # colormaps
from matplotlib.patches import Polygon
from matplotlib.gridspec import GridSpec
# Set up plot aesthetics
plt.clf()
plt.rcdefaults()
colormap = plt.cm.gist_ncar
#color_cycle = [colormap(i) for i in np.linspace(0, 0.9, len(flux_list))]
font_scale = 20
params = {#'backend': .pdf',
'axes.labelsize': font_scale,
'axes.titlesize': font_scale,
'text.fontsize': font_scale,
'legend.fontsize': font_scale*3/4,
'xtick.labelsize': font_scale,
'ytick.labelsize': font_scale,
'font.weight': 500,
'axes.labelweight': 500,
'text.usetex': True,
'figure.figsize': (20, 10),
'figure.titlesize': font_scale
#'axes.color_cycle': color_cycle # colors of different plots
}
plt.rcParams.update(params)
fig = pywcsgrid2.plt.figure()
fig.subplots_adjust(hspace=0.1)
gs1 = GridSpec(1, 2)
#gs1.update(left=0.02, right=0.6, bottom=0.02, top=0.98, wspace=0.00,
# hspace=0.12)
#grid_helper = pywcsgrid2.GridHelper(wcs=cloud_dict[cloud]['av_header'])
grid_pos = 0
cloud_list = []
cloud_dict = {'taurus':cloud_dict['taurus'],
'california':cloud_dict['california']}
for i, cloud in enumerate(cloud_dict):
av_image = cloud_dict[cloud]['av_data']
av_header = cloud_dict[cloud]['av_header']
ax = pywcsgrid2.subplot(gs1[grid_pos],
header=av_header)
# create axes
cmap = cm.pink # colormap
cmap.set_bad(color='w')
# show the image
vmin, vmax = cloud_dict[cloud]['color_scale_limits']
im = ax.imshow(av_image,
interpolation='nearest',
origin='lower',
cmap=cmap,
vmin=vmin,
vmax=vmax,
#norm=matplotlib.colors.LogNorm()
)
# Set limits
limits = cloud_dict[cloud]['limit_pixels']
ax.set_xlim([limits[0][0], limits[1][0]])
ax.set_ylim([limits[0][1], limits[1][1]])
# Asthetics
ax.set_display_coord_system("fk5")
ax.set_ticklabel_type("hms", "dms")
if i != 0:
ax.set_xlabel('')
else:
ax.set_xlabel('Right Ascension (J2000)',)
ax.set_xlabel('Right Ascension (J2000)',)
ax.set_ylabel('Declination (J2000)',)
ax.annotate(cloud.capitalize(),
xy=[0.02, 0.9],
textcoords='axes fraction',
xycoords='axes fraction',
fontsize=font_scale * 1.5,
color='w')
# Convert sky to pix coordinates
cores = cloud_dict[cloud]['cores']
'''
# Colorbar
if cloud == 'perseus':
cb = fig.colorbar(im, ax=ax)
# Write label to colorbar
cb.set_label(r'A$_V$ (Mag)',)
else:
cb = fig.colorbar(im, ax=ax)
# Write label to colorbar
cb.set_label(r'A$_V$ (Mag)',)
'''
for core in cores:
if core in cloud_dict[cloud]['plot_cores']:
scale = 1.1
linewidth=3
linestyle='solid'
else:
scale = 0.8
linewidth=1
linestyle='dashed'
pix_coords = cores[core]['center_pixel']
anno_color = (0.3, 0.5, 1)
ax.scatter(pix_coords[0],pix_coords[1],
color=anno_color,
s=200,
marker='+',
linewidths=2)
ax.annotate(core,
xy=[pix_coords[0], pix_coords[1]],
xytext=(5,10),
textcoords='offset points',
fontsize=font_scale*scale,
#arrowprops=dict(facecolor='w'),
color='w')
if boxes:
vertices = np.copy(cores[core]['box_vertices_rotated'])
#[:, ::-1]
rect = ax.add_patch(Polygon(
vertices[:, ::-1],
facecolor='none',
edgecolor=anno_color,
linewidth=linewidth,
linestyle=linestyle))
cloud_list.append(cloud)
grid_pos += 1
#plt.tight_layout()
if title is not None:
fig.suptitle(title, fontsize=font_scale)
if filename is not None:
plt.savefig(savedir + filename, bbox_inches='tight')
if show:
fig.show()
import pandas as pd
import numpy as np
import os
import errno
import pandas as pd
import torch
from torch import nn
import numpy as np
from sklearn.utils import shuffle
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import matplotlib.pyplot as plt
plt.rcdefaults()
import matplotlib.pyplot as plt
def check_replace(dataframe, column_name):
numbers = []
for element in dataframe[column_name]:
if element not in numbers:
numbers.append(element)
print(numbers)
#def one_hot(dataset, column_num, num_classes): #(data, int, int)
# label = {}
# data_ret = []
# for vec in dataset:
# temp = [0]*num_classes
# temp[int(vec[column_num])] = 1
# new_vec = []
# new_vec.extend(vec[0:column_num])
def multiplot(root, plot_name, nof_coefficients, times, chain_ranks,
star_ranks, chain_size, star_size, rel_diff):
"""
Main plotting interface
:param root: Root folder in which the plot is saved
:param plot_name: Name of the plot
:param nof_coefficients: Number of bath sites
:param times: Time-grid
:param chain_ranks: Bond dimensions in the chain geometry
:param star_ranks: Bond dimensions in the star geometry
:param chain_size: TN-size of the chain
:param star_size: TN-size of the star
:param rel_diff: Relative difference between chain and star
"""
# set global matplotlb rc parameters:
rc = {
'font.family': "serif",
'font.weight': "normal",
'font.size': 11,
'xtick.major.width': 1,
'xtick.major.size': 3,
'ytick.major.width': 1,
'ytick.major.size': 3,
'axes.linewidth': 1,
'legend.frameon': True,
'legend.framealpha': 0
}
plt.figure(1)
plt.rcParams.update(rc)
fig = plt.gcf()
nof_rows, nof_cols = 1, 4
fig_format = 'pdf'
# get some parameters to adjust the suplots
adjust_subplots = True
hspace = 0.45
wspace = 0.6
top = 0.93
bottom = 0.07
left = None
right = None
# tight bbox for the figure:
bbox_inches = 'tight'
# figure width is a DINA4-width, figure height is kept to give the plot-ratio, but maximally it's a DINA4-length
DINA4_length = 29.7
DINA4_width = 21.0
plot_ratio = DINA4_length / (2 * DINA4_width)
in_per_cm = 0.39
# take as figure a folded DINA4-sheet
figwidth = DINA4_width * in_per_cm
figheight = figwidth * plot_ratio * nof_rows / nof_cols
if figheight > DINA4_length * in_per_cm:
figheight = DINA4_length * in_per_cm
# set the variable figsize
figsize = (figwidth, figheight)
# adjust the suplots
if adjust_subplots:
plt.subplots_adjust(hspace=hspace,
wspace=wspace,
top=top,
bottom=bottom,
left=left,
right=right)
plot_2d_xdata = np.arange(nof_coefficients)
vmax = max(np.max(chain_ranks), np.max(star_ranks))
# Plot chain ranks
plt.subplot(nof_rows, nof_cols, 1)
plot_2d(plot_2d_xdata,
times,
chain_ranks,
ylabel='bond',
xlim=(times[0], times[-1]),
ylim=None,
invert_x=False,
invert_y=True,
colorbar=False,
tick_params=None,
colorbar_params=None,
title='(I)',
fontsize_title=11,
vmin=0,
vmax=vmax,
data_transpose=True,
data_flipud=False,
data_fliplr=False,
plotbox_position=None,
fontsize_labels=11)
# Plot star ranks
plt.subplot(nof_rows, nof_cols, 2)
if vmax >= 10:
colorbar_params = {
'orientation':
'vertical',
'aspect':
15,
'pad':
1.8,
'ticks':
[0, int(floor(vmax / 10) * 10) / 2,
int(floor(vmax / 10) * 10)]
}
else:
colorbar_params = {'orientation': 'vertical', 'aspect': 15, 'pad': 1.8}
plot_2d(plot_2d_xdata,
times,
star_ranks,
ylabel=None,
xlim=(times[0], times[-1]),
ylim=None,
invert_x=False,
invert_y=True,
colorbar=True,
tick_params={
'direction': 'in',
'bottom': True,
'top': True,
'left': True,
'right': True,
'labelleft': False
},
colorbar_params=colorbar_params,
title='(II)',
fontsize_title=11,
vmin=0,
vmax=vmax,
data_transpose=True,
data_flipud=False,
data_fliplr=False,
plotbox_position=[0.89, 1, 1, 1],
fontsize_labels=11)
# Plot chain and star size
plt.subplot(nof_rows, nof_cols, 3)
ymin = int(
min(floor(log(np.min(chain_size), 10)),
floor(log(np.min(star_size), 10))))
ymax = int(
max(ceil(log(np.max(chain_size), 10)), ceil(log(np.max(star_size),
10))))
ylim = [10**ymin, 10**ymax]
ytics_tilde = [10**x for x in range(ymin, ymax + 1)]
ytics_labels = ['1e' + str(int(x)) for x in range(ymin, ymax + 1)]
plot_1d(times, ((0, chain_size), (1, star_size)),
ytics_tilde,
ytics_labels,
ylabel=None,
xlim=(times[0], times[-1]),
ylim=ylim,
tick_params=None,
title='size',
fontsize_title=11,
plotbox_position=None,
fontsize_labels=11,
legend=('(I)', '(II)'),
legendloc='best',
fontsize_legend=10,
linewidth=(1.8, 1.8),
linestyle=('-', '--'),
color_list=('blue', 'orange'),
xscale='linear',
yscale='log')
# Plot rel. diff between chain and star
plt.subplot(nof_rows, nof_cols, 4)
ymin = int(floor(log(np.min(rel_diff[rel_diff != 0]), 10)))
ymax = int(ceil(log(np.max(rel_diff), 10)))
ylim = [10**ymin, 10**ymax]
ytics_tilde = [10**x for x in range(ymin, ymax + 1)]
ytics_labels = ['1e' + str(int(x)) for x in range(ymin, ymax + 1)]
if len(ytics_tilde) > 3:
ytics_tilde = ytics_tilde[::2]
if len(ytics_labels) > 3:
ytics_labels = ytics_labels[::2]
plot_1d(times, ((0, rel_diff), ),
ytics_tilde,
yticks_labels=ytics_labels,
ylabel=None,
xlim=(times[0], times[-1]),
ylim=ylim,
tick_params=None,
title='rel. diff.',
fontsize_title=11,
plotbox_position=None,
fontsize_labels=11,
legend=(None, ),
legendloc='best',
fontsize_legend=10,
linewidth=(1.8, ),
linestyle=('-', ),
color_list=('black', ),
xscale='linear',
yscale='log')
# set the figure size of the image
fig.set_size_inches(figsize[0], figsize[1])
# save the figure
plt.savefig(join(root, plot_name + '.' + fig_format),
bbox_inches=bbox_inches)
plt.rcdefaults()
plt.clf() # clear current figure
plt.cla() # clear axes
