def mdl_1d(x, y):
"""builds univariate model to calculate AUC"""
lr = LogisticRegressionCV(scoring='roc_auc')
lars = LassoLarsIC(criterion='aic')
if x.nunique() > 10 and com.is_numeric_dtype(x):
x2 = sb_cutz(x)
series = pd.get_dummies(x2, dummy_na=True)
else:
series = pd.get_dummies(x, dummy_na=True)
lr.fit(series, y)
lars.fit(series, y)
try:
preds = (lr.predict_proba(series)[:, -1])
#preds = (preds > preds.mean()).astype(int)
except ValueError:
Tracer()()
# try:
# cm = confusion_matrix(y, (preds > y.mean()).astype(int))
# except ValueError:
# Tracer()()
aucz = roc_auc_score(y, preds)
ns = num_bin_stats(x, y)
nplot = plot_num(ns)
#plot = plot_confusion_matrix(cm, y)
imgdata = BytesIO()
nplot.savefig(imgdata)
imgdata.seek(0)
nplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdata.getvalue()))
plt.close()
bplot = plot_bubble(ns)
imgdatab = BytesIO()
bplot.savefig(imgdatab)
imgdatab.seek(0)
bplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdatab.getvalue()))
plt.close()
return aucz, nplot, bplot
def mini_histogram(series, **kwargs):
"""Plot a small (mini) histogram of the data.
Parameters
----------
series: Series
The data to plot.
Returns
-------
str
The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, figsize=(2, 0.75), **kwargs)
plot.axes.get_yaxis().set_visible(False)
if LooseVersion(matplotlib.__version__) <= '1.5.9':
plot.set_axis_bgcolor("w")
else:
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def generate_image_placeholder(size, color):
if not size:
size = DEFAULT_SIZE
if not color:
color = DEFAULT_COLOR
cache_key = 'image_placeholder{}'.format(hashlib.md5('{}{}'.format('x'.join(str(size)), color).encode('utf-8')).hexdigest())
img_base64 = cache.get(cache_key)
if img_base64:
return u'data:image/png;base64,{}'.format(img_base64.decode("utf-8"))
bg = struct.unpack('BBB', unhexlify(color.replace('#', '')))
img = Image.new("RGB", (size[0], size[1]), bg)
buffer = BytesIO()
img.save(buffer, format="PNG")
img_base64 = base64.b64encode(buffer.getvalue())
cache.set(cache_key, img_base64, 60*60*24)
return u'data:image/png;base64,{}'.format(img_base64.decode("utf-8"))
def image(filename): gs_file_string = redis.get(filename) buffer_image = BytesIO() gs_image = Image.open(BytesIO(gs_file_string)) gs_image.save(buffer_image, 'JPEG', quality=90) buffer_image.seek(0) return Response(buffer_image.getvalue(), mimetype='image/jpeg')
def mdl_1d_cat(x, y):
"""builds univariate model to calculate AUC"""
if x.nunique() > 10 and com.is_numeric_dtype(x):
x = sb_cutz(x)
series = pd.get_dummies(x, dummy_na=True)
lr = LogisticRegressionCV(scoring='roc_auc')
lr.fit(series, y)
try:
preds = (lr.predict_proba(series)[:, -1])
#preds = (preds > preds.mean()).astype(int)
except ValueError:
Tracer()()
plot = plot_cat(x, y)
imgdata = BytesIO()
plot.savefig(imgdata)
imgdata.seek(0)
aucz = roc_auc_score(y, preds)
cmatrix = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdata.getvalue()))
plt.close()
return aucz, cmatrix
def describe_numeric_1d(series, base_stats):
stats = {'mean': series.mean(), 'std': series.std(), 'variance': series.var(), 'min': series.min(),
'max': series.max()}
stats['range'] = stats['max'] - stats['min']
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats[pretty_name(x)] = series.quantile(x)
stats['iqr'] = stats['75%'] - stats['25%']
stats['kurtosis'] = series.kurt()
stats['skewness'] = series.skew()
stats['sum'] = series.sum()
stats['mad'] = series.mad()
stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
stats['type'] = "NUM"
stats['n_zeros'] = (len(series) - np.count_nonzero(series))
stats['p_zeros'] = stats['n_zeros'] / len(series)
# Large histogram
imgdata = BytesIO()
plot = series.plot(kind='hist', figsize=(6, 4),
facecolor='#337ab7', bins=bins) # TODO when running on server, send this off to a different thread
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(series)
return pd.Series(stats, name=series.name)
def test_dump_xml(self):
try:
expected = unicode(XML_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S'), 'utf-8')
except NameError:
expected = XML_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S')
buf = BytesIO()
dump_xml(self.mc, buf)
result = buf.getvalue().decode('utf-8')
self.assertEqual(expected, result)
def test_dump_js(self):
buf = BytesIO()
dump_js(self.mc, buf)
result = buf.getvalue().decode('utf-8')
try:
JS_RESULT = unicode(JS_RESULT_PYTHON2, 'utf-8') # python 2 add space at the end of lines
except NameError:
JS_RESULT = JS_RESULT_PYTHON3
self.assertEqual(JS_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S'), result)
def render_poster(name_image):
# TODO: Parse for name and base64 Image
name = name_image.split('-')[0]
poster_string = redis.get(name)
buffer_image = BytesIO()
poster_image = Image.open(BytesIO(poster_string))
poster_image.save(buffer_image, 'JPEG', quality=90)
buffer_image.seek(0)
return Response(buffer_image.getvalue(), mimetype='image/jpeg')
def save_poster():
buffer_image = BytesIO()
buffer_image.seek(0)
base64image = request.form['image']
name = request.form['name']
base64image = re.sub('data:image/png;base64,','',str(base64image))
base64image = re.sub('\n','',base64image)
poster_image = Image.open(BytesIO(base64.b64decode(base64image)))
poster_image.save(buffer_image, 'JPEG', quality=90)
buffer_image.seek(0)
redis.set(name, buffer_image.getvalue())
return json.dumps({'success':True}), 200, {'ContentType':'application/json'}
def export(self, request, queryset, tool, content_type, extension='.xlsx'):
try:
from StringIO import BytesIO
except ImportError:
from io import BytesIO #Python3 support
output = BytesIO()
writer = tool(output,
default_date_format=self.export_date_format,
in_memory=True)
writer.perform(*self.resolve_list_fields(queryset),
sheet_name='default')
response = HttpResponse(output.getvalue(), content_type)
filename = self.get_spreadsheet_name(request, queryset, extension)
response['Content-Disposition'] = 'attachment; filename=%s' % filename
writer.close()
return response
def mini_histogram(series):
# Small histogram
imgdata = BytesIO()
plot = series.plot(kind='hist', figsize=(2, 0.75), facecolor='#337ab7', bins=bins)
plot.axes.get_yaxis().set_visible(False)
plot.set_axis_bgcolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def perform(self):
global header
curl = self.curl
if sys.version_info >= (3, ):
buffer = BytesIO()
else:
buffer = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
header = []
try:
curl.perform()
except pycurl.error as pe:
raise occi.TransportError(pe)
# 'Server: Apache/2.2.22 (Debian)\r\n'
h = {}
http_status = None
for item in header:
m = Transport.reHeader.match(item.rstrip())
if m and m.groups >= 2:
key = m.group(1)
value = m.group(2)
h[key.lower()] = value
else:
if Transport.reStatus.match(item):
http_status = item.rstrip()
content_type = None
if 'content-type' in h:
content_type = re.split(';', h['content-type'])[0]
body = buffer.getvalue()
buffer.close()
if sys.version_info >= (3, ):
encoding = 'iso-8859-1'
if content_type:
match = Transport.reEncoding.search(h['content-type'])
if match:
encoding = match.group(1)
body = body.decode(encoding)
return [body, header, http_status, content_type, h]
def perform(self):
global header
curl = self.curl
if sys.version_info >= (3,):
buffer = BytesIO()
else:
buffer = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
header = []
try:
curl.perform()
except pycurl.error as pe:
raise occi.TransportError(pe)
# 'Server: Apache/2.2.22 (Debian)\r\n'
h = {}
http_status = None
for item in header:
m = Transport.reHeader.match(item.rstrip())
if m and m.groups >= 2:
key = m.group(1)
value = m.group(2)
h[key.lower()] = value
else:
if Transport.reStatus.match(item):
http_status = item.rstrip()
content_type = None
if 'content-type' in h:
content_type = re.split(';', h['content-type'])[0]
body = buffer.getvalue()
buffer.close()
if sys.version_info >= (3,):
encoding = 'iso-8859-1'
if content_type:
match = Transport.reEncoding.search(h['content-type'])
if match:
encoding = match.group(1)
body = body.decode(encoding)
return [body, header, http_status, content_type, h]
def mini_histogram(series, **kwargs):
"""Plot a small (mini) histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, figsize=(2, 0.75), **kwargs)
plot.axes.get_yaxis().set_visible(False)
if LooseVersion(matplotlib.__version__) <= '1.5.9':
plot.set_axis_bgcolor("w")
else:
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
try:
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
except:
print('error in setting ticks fontsize')
plot.figure.subplots_adjust(left=0.15,
right=0.85,
top=1,
bottom=0.35,
wspace=0,
hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def test_full(self):
base_path = os.path.dirname(__file__)
tests = [
(load_xml, dump_xml, ["sample_1.xml", "sample_2.xml", "sample_3.xml"]),
(load_json, dump_json, ["sample_1.json", "sample_2.json", "sample_3.json"]),
(load_js, dump_js, ["sample_1.js", "sample_2.js", "sample_3.js"]),
]
for load_fct, dump_fct, files in tests:
for name in files:
path = os.path.join(base_path, name)
with open(path, "rb") as fileobj:
mc = load_fct(fileobj)
buf = BytesIO()
dump_fct(mc, buf)
result = buf.getvalue()
try:
self.assertIsInstance(result.decode("utf-8"), str)
except AssertionError:
self.assertIsInstance(result.decode("utf-8"), unicode) # python 2 backward compatibility
def describe_numeric_1d(series, base_stats):
stats = {
'mean': series.mean(),
'std': series.std(),
'variance': series.var(),
'min': series.min(),
'max': series.max()
}
stats['range'] = stats['max'] - stats['min']
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats[pretty_name(x)] = series.quantile(x)
stats['iqr'] = stats['75%'] - stats['25%']
stats['kurtosis'] = series.kurt()
stats['skewness'] = series.skew()
stats['sum'] = series.sum()
stats['mad'] = series.mad()
stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
stats['type'] = "NUM"
stats['n_zeros'] = (len(series) - np.count_nonzero(series))
stats['p_zeros'] = stats['n_zeros'] / len(series)
# Large histogram
imgdata = BytesIO()
plot = series.plot(
kind='hist', figsize=(6, 4), facecolor='#337ab7', bins=bins
) # TODO when running on server, send this off to a different thread
plot.figure.subplots_adjust(left=0.15,
right=0.95,
top=0.9,
bottom=0.1,
wspace=0,
hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(series)
return pd.Series(stats, name=series.name)
def missing_dendrogram(df):
"""Plot a missingno dendrogram
Parameters
----------
df: DataFrame
The dataframe.
Returns
-------
str
The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = msno.dendrogram(df)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def generate_placeholder_image(size=None):
if not size:
size = DEFAULT_SIZE
cache_key = "image_placeholder_{}".format(
hashlib.md5("x".join(str(size)).encode("utf-8")).hexdigest())
img_base64 = cache.get(cache_key)
if img_base64:
return "data:image/png;base64,{}".format(img_base64)
buffer = BytesIO()
img = Image.new("RGBA", (size[0], size[1]), (255, 0, 0, 100))
img.save(buffer, format="PNG")
img_base64 = base64.b64encode(buffer.getvalue()).decode()
cache.set(cache_key, img_base64, 60 * 60 * 24)
return "data:image/png;base64,{}".format(img_base64)
def histogram(series, **kwargs):
"""Plot an histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, **kwargs)
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
# TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
return result_string
def histogram(series, **kwargs):
"""Plot an histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, **kwargs)
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
# TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
return result_string
def correlation_matrix(corrdf, title, **kwargs):
"""Plot image of a matrix correlation.
Parameters
----------
corrdf: DataFrame
The matrix correlation to plot.
title: str
The matrix title
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
fig_cor, axes_cor = plt.subplots(1, 1)
labels = corrdf.columns
matrix_image = axes_cor.imshow(corrdf,
vmin=-1,
vmax=1,
interpolation="nearest",
cmap='bwr')
plt.title(title, size=18)
plt.colorbar(matrix_image)
num_labels = len(labels)
if num_labels < 1:
return ''
axes_cor.set_xticks(
np.arange(0, corrdf.shape[0], corrdf.shape[0] * 1.0 / num_labels))
axes_cor.set_yticks(
np.arange(0, corrdf.shape[1], corrdf.shape[1] * 1.0 / num_labels))
axes_cor.set_xticklabels(labels, rotation=90)
axes_cor.set_yticklabels(labels)
matrix_image.figure.savefig(imgdata, bbox_inches='tight')
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
plt.close(matrix_image.figure)
return result_string
def complete_histogram(hist_data):
"""Large histogram"""
img_data = BytesIO()
plt.figure(figsize=(6, 4))
plot = plt.subplot()
plt.bar(hist_data['left_edge'],
hist_data['count'],
width=hist_data['width'],
facecolor='#337ab7')
plot.set_ylabel('Frequency')
plot.figure.subplots_adjust(left=0.15,
right=0.95,
top=0.9,
bottom=0.1,
wspace=0,
hspace=0)
plot.figure.savefig(img_data)
img_data.seek(0)
result_string = BASE + quote(base64.b64encode(img_data.getvalue()))
# TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
return result_string
def correlation_matrix(corrdf, title, **kwargs):
"""Plot image of a matrix correlation.
Parameters
----------
corrdf: DataFrame
The matrix correlation to plot.
title: str
The matrix title
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
fig_cor, axes_cor = plt.subplots(1, 1)
labels = corrdf.columns
N = 256
blues = np.ones((N, 4))
blues[:, 0] = np.linspace(1, 66/256, N)
blues[:, 1] = np.linspace(1, 136/256, N)
blues[:, 2] = np.linspace(1, 181/256, N)
reds = np.ones((N, 4))
reds[:, 0] = np.linspace(209/256, 1, N)
reds[:, 1] = np.linspace(60/256, 1, N)
reds[:, 2] = np.linspace(75/256, 1, N)
newcmp = ListedColormap(np.concatenate((reds, blues)))
matrix_image = axes_cor.imshow(corrdf, vmin=-1, vmax=1, interpolation="nearest", cmap=newcmp)
plt.title(title, size=18)
plt.colorbar(matrix_image)
axes_cor.set_xticks(np.arange(0, corrdf.shape[0], corrdf.shape[0] * 1.0 / len(labels)))
axes_cor.set_yticks(np.arange(0, corrdf.shape[1], corrdf.shape[1] * 1.0 / len(labels)))
axes_cor.set_xticklabels(labels, rotation=90)
axes_cor.set_yticklabels(labels)
matrix_image.figure.savefig(imgdata, bbox_inches='tight')
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(matrix_image.figure)
return result_string
def request_data(start_time=None, date=None, start=None, end=None):
header = create_header()
t = time.localtime()
if date is None:
date = time.strftime("%Y-%m-%d", t)
if start_time is None:
start_time = time.strftime("%H:%M:%S", t)
if start is None:
#haltestellenid = Trinidadstr
start = '1:000009014238'
if end is None:
#Haltestellenid = Bahnhof Sebaldsbrück
end = '1:000009013744'
buffer = BytesIO()
c = pycurl.Curl()
address = 'http://gtfsr.vbn.de/api/'
address += ('routers/connect/plan?arriveBy=false'
'time={0}'
'&date={1}'
'&fromPlace={2}'
'&toPlace={3}').format(start_time, date, start, end)
c.setopt(c.URL, address)
c.setopt(c.HTTPHEADER, header)
c.setopt(c.WRITEDATA, buffer)
with Timeout(5, False):
try:
c.perform()
c.close()
except Exception as e:
print('exeption: ')
print(e)
return [' '] * 4
body = buffer.getvalue().decode('UTF8')
return prepare_data(body)
def mini_histogram(histogram_data):
# Small histogram
imgdata = BytesIO()
hist_data = histogram_data
figure = plt.figure(figsize=(2, 0.75))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.axes.get_yaxis().set_visible(False)
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def mini_histogram(histogram_data):
# Small histogram
imgdata = BytesIO()
hist_data = histogram_data
figure = plt.figure(figsize=(2, 0.75))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.axes.get_yaxis().set_visible(False)
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def mini_histogram(series):
# Small histogram
imgdata = BytesIO()
plot = series.plot(kind='hist', figsize=(2, 0.75), facecolor='#337ab7')
plot.axes.get_yaxis().set_visible(False)
plot.set_axis_bgcolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15,
right=0.85,
top=1,
bottom=0.35,
wspace=0,
hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def describe_float_1d(df, column, current_result, nrows):
if spark_version == "1.6+":
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
variance(col(column)).alias("variance"),
kurtosis(col(column)).alias("kurtosis"),
stddev(col(column)).alias("std"),
skewness(col(column)).alias("skewness"),
df_sum(col(column)).alias("sum"),
count(col(column) == 0.0).alias('n_zeros')
).toPandas()
else:
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
df_sum(col(column)).alias("sum"),
count(col(column) == 0.0).alias('n_zeros')
).toPandas()
stats_df["variance"] = df.select(column).na.drop().agg(variance_custom(col(column),
stats_df["mean"].iloc[0],
current_result["count"])).toPandas().iloc[0][0]
stats_df["std"] = np.sqrt(stats_df["variance"])
stats_df["skewness"] = df.select(column).na.drop().agg(skewness_custom(col(column),
stats_df["mean"].iloc[0],
current_result["count"])).toPandas().iloc[0][0]
stats_df["kurtosis"] = df.select(column).na.drop().agg(kurtosis_custom(col(column),
stats_df["mean"].iloc[0],
current_result["count"])).toPandas().iloc[0][0]
for x in [0.05, 0.25, 0.5, 0.75, 0.95]:
stats_df[pretty_name(x)] = (df.select(column)
.na.drop()
.selectExpr("percentile_approx(`{col}`,CAST({n} AS DOUBLE))"
.format(col=column, n=x)).toPandas().iloc[:,0]
)
stats = stats_df.iloc[0].copy()
stats.name = column
stats["range"] = stats["max"] - stats["min"]
stats["iqr"] = stats[pretty_name(0.75)] - stats[pretty_name(0.25)]
stats["cv"] = stats["std"] / float(stats["mean"])
stats["mad"] = (df.select(column)
.na.drop()
.select(df_abs(col(column)-stats["mean"]).alias("delta"))
.agg(df_sum(col("delta"))).toPandas().iloc[0,0] / float(current_result["count"]))
stats["type"] = "NUM"
stats['p_zeros'] = stats['n_zeros'] / float(nrows)
# Large histogram
imgdata = BytesIO()
hist_data = create_hist_data(df, column, stats["min"], stats["max"], bins)
figure = plt.figure(figsize=(6, 4))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.set_ylabel("Frequency")
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(hist_data)
return stats
def occi_curl(base_url=None, url='/-/', authtype=None, ignoressl=None, user=None, passwd=None, mimetype=None, headers=[], post='', custom_request=''):
"""Send HTTP request
:param string base_url: OCCI server URL (default: from config)
:param string url: path element of the URL
:param string authtype: authentication type (default: from config)
:param bool ignoressl: ignore SSL problems (default: from config)
:param string user: user name for 'basic' auth (default: from config)
:param string passwd: password for 'basic' auth (default: from config)
:param string mimetype: accepted mimetype (empty string='\*/\*')
:param string headers[]: HTTP Headers
:param string post: HTTP Body
:param string custom_request: HTTP Request type (default: 'GET' or 'POST')
:return: [body, header, HTTP status, content type]
:rtype: [string[], string[], string, string]
"""
global header
if base_url is None:
base_url = occi_config['url']
if authtype is None:
authtype = occi_config['authtype']
if ignoressl is None:
ignoressl = occi_config['ignoressl']
if user is None:
user = occi_config['user']
if passwd is None:
passwd = occi_config['passwd']
if mimetype is None:
mimetype = occi_config['mimetype']
curlverbose = occi_config['curlverbose']
if sys.version_info >= (3,):
buffer = BytesIO()
else:
buffer = StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, str(base_url + url))
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
# Disable check of SSL certificate
if ignoressl:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if 'capath' in occi_config and occi_config['capath']:
curl.setopt(pycurl.CAPATH, occi_config['capath'])
if 'cachain' in occi_config and occi_config['cachain']:
curl.setopt(pycurl.CAINFO, occi_config['cachain'])
# Name and password for basic auth (ONLY SUPPORTED YET)
if authtype == "basic":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.USERPWD, "%s:%s" % (user, passwd))
elif authtype == "x509":
if 'cert' in occi_config and occi_config['cert']:
curl.setopt(pycurl.SSLCERT, occi_config['cert'])
if 'key' in occi_config and occi_config['key']:
curl.setopt(pycurl.SSLKEY, occi_config['key'])
if 'passphrase' in occi_config and occi_config['passphrase']:
curl.setopt(pycurl.SSLCERTPASSWD, occi_config['passphrase'])
# Verbose mode
curl.setopt(pycurl.VERBOSE, curlverbose)
curl.setopt(pycurl.CONNECTTIMEOUT, occi_config['connectiontimeout'])
curl.setopt(pycurl.TIMEOUT, occi_config['timeout'])
# Set appropriate mime type
if mimetype:
headers = ['Accept: %s' % mimetype] + headers
else:
headers = ['Accept: */*'] + headers
# Set requested HTTP headers
if headers:
curl.setopt(pycurl.HTTPHEADER, headers)
# HTTP header response
if sys.version_info >= (3,):
curl.setopt(pycurl.HEADERFUNCTION, get_header3)
else:
curl.setopt(pycurl.HEADERFUNCTION, get_header2)
if post or custom_request == 'POST':
curl.setopt(pycurl.POST, 1)
if post:
curl.setopt(pycurl.POSTFIELDS, post)
else:
curl.setopt(pycurl.POSTFIELDS, 'OK')
if curlverbose:
print "==== POST ==== "
print post
print "============== "
if custom_request and custom_request != 'POST':
curl.setopt(pycurl.CUSTOMREQUEST, custom_request)
# DO IT!
header = []
try:
curl.perform()
curl.close()
except pycurl.error as pe:
raise occi.Error(pe)
## 'Server: Apache/2.2.22 (Debian)\r\n'
h = {}
for item in header:
if re.match(r'.*:.*', item):
key = re.sub(r':.*', r'', item.rstrip())
value = re.sub(r'([^:]*):\s*(.*)', r'\2', item.rstrip())
h[key] = value
else:
if re.match(r'^HTTP', item):
http_status = item.rstrip()
content_type = None
if 'Content-Type' in h:
content_type = re.split(';', h['Content-Type'])[0]
body = buffer.getvalue()
if sys.version_info >= (3,):
encoding = 'iso-8859-1'
if content_type:
match = re.search(r';\s*charset=(\S+)', h['Content-Type'])
if match:
encoding = match.group(1)
body = body.decode(encoding)
return [body.splitlines(), header, http_status, content_type]
def describe_float_1d(df, column, current_result, nrows):
if spark_version == "1.6+":
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
variance(col(column)).alias("variance"),
kurtosis(col(column)).alias("kurtosis"),
stddev(col(column)).alias("std"),
skewness(col(column)).alias("skewness"),
df_sum(col(column)).alias("sum")
).toPandas()
else:
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
df_sum(col(column)).alias("sum")
).toPandas()
stats_df["variance"] = df.select(column).na.drop().agg(variance_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
stats_df["std"] = np.sqrt(stats_df["variance"])
stats_df["skewness"] = df.select(column).na.drop().agg(skewness_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
stats_df["kurtosis"] = df.select(column).na.drop().agg(kurtosis_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats_df[pretty_name(x)] = (df.select(column)
.na.drop()
.selectExpr("percentile_approx(`{col}`,CAST({n} AS DOUBLE))"
.format(col=column, n=x)).toPandas().ix[:,0]
)
stats = stats_df.ix[0].copy()
stats.name = column
stats["range"] = stats["max"] - stats["min"]
stats["iqr"] = stats[pretty_name(0.75)] - stats[pretty_name(0.25)]
stats["cv"] = stats["std"] / float(stats["mean"])
stats["mad"] = (df.select(column)
.na.drop()
.select(df_abs(col(column)-stats["mean"]).alias("delta"))
.agg(df_sum(col("delta"))).toPandas().ix[0,0] / float(current_result["count"]))
stats["type"] = "NUM"
stats['n_zeros'] = df.select(column).where(col(column)==0.0).count()
stats['p_zeros'] = stats['n_zeros'] / float(nrows)
# Large histogram
imgdata = BytesIO()
hist_data = create_hist_data(df, column, stats["min"], stats["max"], bins)
figure = plt.figure(figsize=(6, 4))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.set_ylabel("Frequency")
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(hist_data)
return stats
def point_cloud_to_buffer(pc, data_compression=None):
fileobj = BytesIO()
point_cloud_to_fileobj(pc, fileobj, data_compression)
return fileobj.getvalue()
def create_pdf(form=None, sig_image=None):
"""
given form results -
"""
def get_from_form(value):
if form:
return form.cleaned_data.get(value, "")
else:
return ""
email = get_from_form("email")
phone_number = get_from_form("phone")
postcode = get_from_form("postcode")
surname = get_from_form("surname")
first_names = get_from_form("first_names")
add_1 = get_from_form("add_1")
add_2 = get_from_form("add_2")
city = get_from_form("city")
county = get_from_form("county")
alt_add_1 = get_from_form("alt_add_1")
alt_add_2 = get_from_form("alt_add_2")
alt_postcode = get_from_form("alt_postcode")
alt_reason = get_from_form("reason")
file_name = file_name_safe("{0}_{1}".format(surname, first_names).lower())
council = get_from_form("council")
multi_council = get_from_form("multi_council")
if city and county:
add_3 = city + ", " + county
elif city:
add_3 = city
elif county:
add_3 = county
until_further_notice = get_from_form("universal")
one_date = get_from_form("single_day")
date_range = get_from_form("time_range")
date_of_birth = get_from_form("dob")
packet = BytesIO()
# create a new PDF with Reportlab
can = canvas.Canvas(packet, pagesize=letter)
# add signature of present
if sig_image:
can.drawImage(ImageReader(sig_image), 293, 155, mask='auto')
# core address info
can.drawString(40, 667, surname.upper())
can.drawString(40, 620, first_names.upper())
can.drawString(40, 561, add_1.upper())
can.drawString(40, 541, add_2.upper())
can.drawString(40, 521, add_3.upper())
can.drawString(40, 390, email.upper())
can.drawString(40, 451, phone_number)
can.drawString(100, 499, postcode.upper())
# alt address
can.drawString(285, 646, alt_add_1.upper())
can.drawString(285, 626, alt_add_2.upper())
can.drawString(350, 606, alt_postcode.upper())
can.drawString(285, 548, alt_reason.upper())
# for how long we want this on
if until_further_notice:
can.drawString(30, 278, "X")
if one_date:
can.drawString(30, 248, "X")
write_date(can.drawString, 153, 213, one_date)
if date_range:
can.drawString(30, 181, "X")
write_date(can.drawString, 153, 156, date_range[0])
write_date(can.drawString, 153, 129, date_range[1])
# today's date
write_date(can.drawString, 457, 44, datetime.datetime.now())
# birthdate
can.setFont("Helvetica", 30)
write_date(can.drawString,
310,
350,
date_of_birth,
25,
extra_spacing=[1, 3])
can.save()
packet.seek(0)
new_pdf = PdfFileReader(packet)
front_page = create_front_page(council, postcode, multi_council)
source_file = os.path.join(settings.PROJECT_PATH, "resources", "form.pdf")
existing_pdf = PdfFileReader(open(source_file, "rb"))
output = PdfFileWriter()
# add the several pages objects to one pdf
page = existing_pdf.getPage(0)
page.mergePage(new_pdf.getPage(0))
output.addPage(front_page)
output.addPage(page)
# send the stream into a response and return it to the view
outputStream = BytesIO()
output.write(outputStream)
response = HttpResponse(content_type='application/pdf')
response[
'Content-Disposition'] = 'attachment; filename="postal_vote_{0}.pdf"'.format(
file_name)
response.write(outputStream.getvalue())
outputStream.close()
if council:
council.increment_count()
return response
