def mini_histogram(series, **kwargs):
"""Plot a small (mini) histogram of the data.
Parameters
----------
series: Series
The data to plot.
Returns
-------
str
The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, figsize=(2, 0.75), **kwargs)
plot.axes.get_yaxis().set_visible(False)
if LooseVersion(matplotlib.__version__) <= '1.5.9':
plot.set_axis_bgcolor("w")
else:
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def describe_numeric_1d(series, base_stats):
stats = {'mean': series.mean(), 'std': series.std(), 'variance': series.var(), 'min': series.min(),
'max': series.max()}
stats['range'] = stats['max'] - stats['min']
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats[pretty_name(x)] = series.quantile(x)
stats['iqr'] = stats['75%'] - stats['25%']
stats['kurtosis'] = series.kurt()
stats['skewness'] = series.skew()
stats['sum'] = series.sum()
stats['mad'] = series.mad()
stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
stats['type'] = "NUM"
stats['n_zeros'] = (len(series) - np.count_nonzero(series))
stats['p_zeros'] = stats['n_zeros'] / len(series)
# Large histogram
imgdata = BytesIO()
plot = series.plot(kind='hist', figsize=(6, 4),
facecolor='#337ab7', bins=bins) # TODO when running on server, send this off to a different thread
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(series)
return pd.Series(stats, name=series.name)
def image(filename): gs_file_string = redis.get(filename) buffer_image = BytesIO() gs_image = Image.open(BytesIO(gs_file_string)) gs_image.save(buffer_image, 'JPEG', quality=90) buffer_image.seek(0) return Response(buffer_image.getvalue(), mimetype='image/jpeg')
def to_greyscale(profile_picture):
response = requests.get(profile_picture['source'])
gs_image = Image.open(BytesIO(response.content)).convert('L')
buffer_image = BytesIO()
gs_image.save(buffer_image, 'JPEG', quality=90)
buffer_image.seek(0)
return buffer_image
def mdl_1d_cat(x, y):
"""builds univariate model to calculate AUC"""
if x.nunique() > 10 and com.is_numeric_dtype(x):
x = sb_cutz(x)
series = pd.get_dummies(x, dummy_na=True)
lr = LogisticRegressionCV(scoring='roc_auc')
lr.fit(series, y)
try:
preds = (lr.predict_proba(series)[:, -1])
#preds = (preds > preds.mean()).astype(int)
except ValueError:
Tracer()()
plot = plot_cat(x, y)
imgdata = BytesIO()
plot.savefig(imgdata)
imgdata.seek(0)
aucz = roc_auc_score(y, preds)
cmatrix = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdata.getvalue()))
plt.close()
return aucz, cmatrix
def generate_image_placeholder(size, color):
if not size:
size = DEFAULT_SIZE
if not color:
color = DEFAULT_COLOR
cache_key = 'image_placeholder{}'.format(hashlib.md5('{}{}'.format('x'.join(str(size)), color).encode('utf-8')).hexdigest())
img_base64 = cache.get(cache_key)
if img_base64:
return u'data:image/png;base64,{}'.format(img_base64.decode("utf-8"))
bg = struct.unpack('BBB', unhexlify(color.replace('#', '')))
img = Image.new("RGB", (size[0], size[1]), bg)
buffer = BytesIO()
img.save(buffer, format="PNG")
img_base64 = base64.b64encode(buffer.getvalue())
cache.set(cache_key, img_base64, 60*60*24)
return u'data:image/png;base64,{}'.format(img_base64.decode("utf-8"))
def test_dump_xml(self):
try:
expected = unicode(XML_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S'), 'utf-8')
except NameError:
expected = XML_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S')
buf = BytesIO()
dump_xml(self.mc, buf)
result = buf.getvalue().decode('utf-8')
self.assertEqual(expected, result)
def test_dump_js(self):
buf = BytesIO()
dump_js(self.mc, buf)
result = buf.getvalue().decode('utf-8')
try:
JS_RESULT = unicode(JS_RESULT_PYTHON2, 'utf-8') # python 2 add space at the end of lines
except NameError:
JS_RESULT = JS_RESULT_PYTHON3
self.assertEqual(JS_RESULT % self.mc.creation.strftime('%Y-%m-%d %H:%M:%S'), result)
def render_poster(name_image):
# TODO: Parse for name and base64 Image
name = name_image.split('-')[0]
poster_string = redis.get(name)
buffer_image = BytesIO()
poster_image = Image.open(BytesIO(poster_string))
poster_image.save(buffer_image, 'JPEG', quality=90)
buffer_image.seek(0)
return Response(buffer_image.getvalue(), mimetype='image/jpeg')
def mini_histogram(series):
# Small histogram
imgdata = BytesIO()
plot = series.plot(kind='hist', figsize=(2, 0.75), facecolor='#337ab7', bins=bins)
plot.axes.get_yaxis().set_visible(False)
plot.set_axis_bgcolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def perform(self):
global header
curl = self.curl
if sys.version_info >= (3,):
buffer = BytesIO()
else:
buffer = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
header = []
try:
curl.perform()
except pycurl.error as pe:
raise occi.TransportError(pe)
# 'Server: Apache/2.2.22 (Debian)\r\n'
h = {}
http_status = None
for item in header:
m = Transport.reHeader.match(item.rstrip())
if m and m.groups >= 2:
key = m.group(1)
value = m.group(2)
h[key.lower()] = value
else:
if Transport.reStatus.match(item):
http_status = item.rstrip()
content_type = None
if 'content-type' in h:
content_type = re.split(';', h['content-type'])[0]
body = buffer.getvalue()
buffer.close()
if sys.version_info >= (3,):
encoding = 'iso-8859-1'
if content_type:
match = Transport.reEncoding.search(h['content-type'])
if match:
encoding = match.group(1)
body = body.decode(encoding)
return [body, header, http_status, content_type, h]
def test_xls_output_stringio(self):
data = [[1, 2, 3], [4, 5, 6]]
io = BytesIO()
w = pe.Writer(("xls", io))
w.write_rows(data)
w.close()
r = pe.Reader(("xls", io.getvalue()))
result = [1, 2, 3, 4, 5, 6]
actual = pe.utils.to_array(r.enumerate())
assert result == actual
def dump(self, obj):
"""
Dumps the given object in the Java serialization format
"""
self.references = []
self.object_obj = obj
self.object_stream = BytesIO()
self._writeStreamHeader()
self.writeObject(obj)
return self.object_stream.getvalue()
def mini_histogram(series, **kwargs):
"""Plot a small (mini) histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, figsize=(2, 0.75), **kwargs)
plot.axes.get_yaxis().set_visible(False)
if LooseVersion(matplotlib.__version__) <= '1.5.9':
plot.set_axis_bgcolor("w")
else:
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
try:
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
except:
print('error in setting ticks fontsize')
plot.figure.subplots_adjust(left=0.15,
right=0.85,
top=1,
bottom=0.35,
wspace=0,
hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def create_b64thumb(image):
thumb = None
im = Image.open(image)
#im.thumbnail((480,270))
im = ImageOps.fit(im, (480, 270), centering=(0.0, 0.0))
#tmp = StringIO()
tmp = BytesIO()
im.save(tmp, format="PNG")
thumb = base64.b64encode(tmp.getvalue())
return thumb
def retrieve_compressed_data(url):
if url:
request = urllib.Request(url)
opener = urllib.build_opener()
response = opener.open(request)
gz_data = BytesIO(response.read())
gzipper = gzip.GzipFile(fileobj=gz_data)
data = gzipper.read()
return data.decode('utf-8').strip().split('\n')
return []
def test_write_varint(self):
expected = b('\xac\x02')
outfile = BytesIO()
written = varblock.write_varint(outfile, 300)
# Test the returned values
self.assertEqual(expected, outfile.getvalue())
self.assertEqual(2, written)
def get(self, package):
self.set_header("Content-Type", "image/png")
url = PYPI_URL % package
license = self.get_license(url)
license = license.replace(' ', '_')
shield_url = SHIELD_URL % ("license", license, "blue")
shield = requests.get(shield_url).content
img = BytesIO(shield)
img.seek(0)
self.write(img.read())
def describe_numeric_1d(series, base_stats):
stats = {
'mean': series.mean(),
'std': series.std(),
'variance': series.var(),
'min': series.min(),
'max': series.max()
}
stats['range'] = stats['max'] - stats['min']
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats[pretty_name(x)] = series.quantile(x)
stats['iqr'] = stats['75%'] - stats['25%']
stats['kurtosis'] = series.kurt()
stats['skewness'] = series.skew()
stats['sum'] = series.sum()
stats['mad'] = series.mad()
stats['cv'] = stats['std'] / stats['mean'] if stats['mean'] else np.NaN
stats['type'] = "NUM"
stats['n_zeros'] = (len(series) - np.count_nonzero(series))
stats['p_zeros'] = stats['n_zeros'] / len(series)
# Large histogram
imgdata = BytesIO()
plot = series.plot(
kind='hist', figsize=(6, 4), facecolor='#337ab7', bins=bins
) # TODO when running on server, send this off to a different thread
plot.figure.subplots_adjust(left=0.15,
right=0.95,
top=0.9,
bottom=0.1,
wspace=0,
hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(
base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(series)
return pd.Series(stats, name=series.name)
def test_full(self):
base_path = os.path.dirname(__file__)
tests = [
(load_xml, dump_xml, ["sample_1.xml", "sample_2.xml", "sample_3.xml"]),
(load_json, dump_json, ["sample_1.json", "sample_2.json", "sample_3.json"]),
(load_js, dump_js, ["sample_1.js", "sample_2.js", "sample_3.js"]),
]
for load_fct, dump_fct, files in tests:
for name in files:
path = os.path.join(base_path, name)
with open(path, "rb") as fileobj:
mc = load_fct(fileobj)
buf = BytesIO()
dump_fct(mc, buf)
result = buf.getvalue()
try:
self.assertIsInstance(result.decode("utf-8"), str)
except AssertionError:
self.assertIsInstance(result.decode("utf-8"), unicode) # python 2 backward compatibility
def processRequest(url, usernamepassword, data, method, isHttps, certfile,
isDebug):
buffer = BytesIO()
header = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, url)
c.setopt(pycurl.HTTPHEADER,
['X-Requested-By: ambari', 'Content-Type: application/json'])
c.setopt(pycurl.USERPWD, usernamepassword)
c.setopt(pycurl.VERBOSE, 0)
if isHttps == True:
c.setopt(pycurl.SSL_VERIFYPEER, 1)
c.setopt(pycurl.SSL_VERIFYHOST, 2)
c.setopt(pycurl.CAINFO, certfile)
c.setopt(c.WRITEFUNCTION, buffer.write)
c.setopt(c.HEADERFUNCTION, header.write)
# setting proper method and parameters
if method == 'get':
c.setopt(pycurl.HTTPGET, 1)
elif method == 'delete':
c.setopt(pycurl.CUSTOMREQUEST, "DELETE")
c.setopt(c.POSTFIELDS, str(data))
else:
log(
"[E] Unknown Http Request method found, only get or delete method are allowed!",
"error")
c.perform()
# getting response
response = buffer.getvalue()
headerResponse = header.getvalue()
response_code = 0
response_code = str(c.getinfo(pycurl.RESPONSE_CODE))
response_code = int(response_code)
buffer.close()
header.close()
c.close()
if isDebug == True or (response_code != 200 and response_code != 204):
print 'Request URL = ' + str(url)
print 'Response = ' + str(headerResponse)
return response_code
def scan_site(self,sitemap_url):
sitemap = urlopen(sitemap_url).read()
urls = etree.iterparse(BytesIO(sitemap), tag='{http://www.sitemaps.org/schemas/sitemap/0.9}url')
NS = {
'x': 'http://www.sitemaps.org/schemas/sitemap/0.9',
'x2': 'http://www.google.com/schemas/sitemap-mobile/1.0'
}
for event, url in urls:
search_url = url.xpath('.//x:loc/text()', namespaces=NS)[0]
self._parse_url(search_url)
def download_blitz_target():
loc = get_blitz_location()
if "ZIP_FILE" in os.environ:
zipfile = ZipFile(loc)
else:
print("Downloading %s ..." % loc, file=sys.stderr)
resp = urlopen(loc)
content = resp.read()
content = BytesIO(content)
zipfile = ZipFile(content)
zipfile.extractall("target")
def render_latex(formula, fontsize=12, dpi=300, format_='svg'):
"""Renders LaTeX formula into image."""
fig = plt.figure()
text = fig.text(0, 0, u'${0}$'.format(formula), fontsize=fontsize)
fig.savefig(BytesIO(), dpi=dpi) # triggers rendering
bbox = text.get_window_extent()
width, height = bbox.size / float(dpi) + 0.05
fig.set_size_inches((width, height))
dy = (bbox.ymin / float(dpi)) / height
text.set_position((0, -dy))
buffer_ = BytesIO()
fig.savefig(buffer_, dpi=dpi, transparent=True, format=format_)
plt.close(fig)
buffer_.seek(0)
return buffer_
def get(self, package):
self.set_header("Content-Type", "image/png")
url = PYPI_URL % package
has_wheel = self.get_wheel(url)
wheel_text = "yes" if has_wheel else "no"
colour = "green" if has_wheel else "red"
shield_url = SHIELD_URL % ("wheel", wheel_text, colour)
shield = requests.get(shield_url).content
img = BytesIO(shield)
img.seek(0)
self.write(img.read())
def test_correct(self):
# Unsigned byte.
ndarr = np.array([0x0A, 0x0B, 0xFF], dtype='uint8')
with contextlib.closing(BytesIO()) as bytesio:
idx2numpy.convert_to_file(bytesio, ndarr)
self.assertEqual(bytesio.getvalue(),
b'\x00\x00\x08\x01\x00\x00\x00\x03' +
b'\x0A' +
b'\x0B' +
b'\xFF')
def getRemoteData(self, endpoint):
curl = pycurl.Curl()
buffer = BytesIO()
curl.setopt(pycurl.URL, endpoint)
curl.setopt(pycurl.TIMEOUT, 10)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(curl.WRITEDATA, buffer)
curl.perform()
return buffer.getvalue()
def missing_dendrogram(df):
"""Plot a missingno dendrogram
Parameters
----------
df: DataFrame
The dataframe.
Returns
-------
str
The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = msno.dendrogram(df)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def get(self, package):
self.set_header("Content-Type", "image/png")
url = PYPI_URL % package
has_egg = self.get_egg(url)
egg_text = "yes" if has_egg else "no"
colour = "red" if has_egg else "brightgreen"
shield_url = SHIELD_URL % ("egg", egg_text, colour)
shield = requests.get(shield_url).content
img = BytesIO(shield)
img.seek(0)
self.write(img.read())
def callPyCurlRequest(self, url, data, method, usernamepassword):
buffer = BytesIO()
header = BytesIO()
logger.info(url)
# Creating PyCurl Requests
c = pycurl.Curl()
c.setopt(c.URL, url)
#c.setopt(pycurl.HTTPHEADER, ['Content-Type: application/json','Accept: application/json'])
c.setopt(pycurl.USERPWD, usernamepassword)
c.setopt(pycurl.VERBOSE, 0)
c.setopt(pycurl.SSL_VERIFYPEER, False)
c.setopt(pycurl.SSL_VERIFYHOST, False)
c.setopt(c.WRITEFUNCTION, buffer.write)
c.setopt(c.HEADERFUNCTION, header.write)
# setting proper method and parameters
if method == 'get':
c.setopt(pycurl.HTTPGET, 1)
elif method == 'post':
c.setopt(c.POSTFIELDS, data)
elif method == 'put':
c.setopt(pycurl.CUSTOMREQUEST, "PUT")
c.setopt(c.POSTFIELDS, str(data))
elif method == 'delete':
c.setopt(pycurl.CUSTOMREQUEST, "DELETE")
c.setopt(c.POSTFIELDS, str(data))
else:
print('xa_ambari_api_util callCurlRequest : method is not get, post, put or delete')
# making request
c.perform()
# getting response
response = buffer.getvalue()
headerResponse = header.getvalue()
c.close()
buffer.close()
header.close()
return headerResponse, response
def histogram(series, **kwargs):
"""Plot an histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, **kwargs)
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
# TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
return result_string
def loads(string):
"""
Deserializes Java objects and primitive data serialized using
ObjectOutputStream from a string.
:param string: A Java data string
:return: The deserialized object
"""
file_like = BytesIO(string)
marshaller = JavaObjectUnmarshaller(file_like)
marshaller.add_transformer(DefaultObjectTransformer())
return marshaller.readObject()
def generate_placeholder_image(size=None):
if not size:
size = DEFAULT_SIZE
cache_key = "image_placeholder_{}".format(
hashlib.md5("x".join(str(size)).encode("utf-8")).hexdigest())
img_base64 = cache.get(cache_key)
if img_base64:
return "data:image/png;base64,{}".format(img_base64)
buffer = BytesIO()
img = Image.new("RGBA", (size[0], size[1]), (255, 0, 0, 100))
img.save(buffer, format="PNG")
img_base64 = base64.b64encode(buffer.getvalue()).decode()
cache.set(cache_key, img_base64, 60 * 60 * 24)
return "data:image/png;base64,{}".format(img_base64)
def _profile2dict(lines, fieldnames):
import re
data = np.recfromtxt(
BytesIO(
bytes(re.sub(r'[ ]+"', '"', '\n'.join(lines)).replace('"', ''),
encoding='ascii')),
delimiter=' ',
names=fieldnames,
converters=dict(names=lambda x: x.strip()),
)
return data
def test_put_file_overwrite(self):
"""Tests if put_file with overwrite=true returns the expected metadata"""
path = posixpath.join(self.test_dir, "foo_overwrite.txt")
self.upload_file(self.foo, path)
f = BytesIO(b"This Overwrites")
metadata = self.client.put_file(path, f, overwrite=True)
self.dict_has(metadata,
size="15 bytes",
bytes=15,
is_dir=False,
path=path,
mime_type="text/plain")
def loads(data, charset='utf-8', errors=default_errors, decode_strings=False,
object_hook=None, array_hook=None, return_unicode=False):
"""Read a PHP-serialized object hierarchy from a string. Characters in the
string past the object's representation are ignored. On Python 3 the
string must be a bytestring.
"""
# Convert unicode strings to byte strings.
if type(data) == unicode:
data = data.encode(charset)
return_unicode = True
return load(BytesIO(data), charset, errors, decode_strings,
object_hook, array_hook, return_unicode)
def compile_sprite_frames(self, imagePath, frameSize):
# image does not exist
if not os.path.exists(imagePath):
print("error")
print("\t----> sprite '%s' not found" % spriteFilename)
return None
# attempt to open sprite
try:
sprite = Image.open(imagePath)
except IOError as detail:
print("error")
print(" - ----> %s" % detail)
return None
# invalid mode
if sprite.mode not in ["LA", "RGBA", "RGB"]:
print("error")
print("\t----> sprite uses incompatible color mode")
return None
# output buffer
outputBuffer = b""
# get size of frames
frameSize = list(map(int, frameSize))
for i in frameSize:
outputBuffer += struct.pack("<H", int(i))
# write frame count
outputBuffer += struct.pack(
"<H",
int((sprite.size[1] / frameSize[1]) *
(sprite.size[0] / frameSize[0])))
# compile tile data
for y in range(int(sprite.size[1] / frameSize[1])):
if sprite.size[1] - (y * frameSize[1]) < frameSize[1]: break
for x in range(int(sprite.size[0] / frameSize[0])):
if sprite.size[0] - (x * frameSize[0]) < frameSize[0]: continue
tile = sprite.copy().crop(
(x * frameSize[0], y * frameSize[1],
(x + 1) * frameSize[0], (y + 1) * frameSize[1]))
tileData = BytesIO()
tile.convert("RGBA").save(tileData, "PNG")
outputBuffer += struct.pack("<L", len(tileData.getvalue()))
outputBuffer += tileData.getvalue()
tileData.close()
return outputBuffer
def serialize(elem, **options):
file = BytesIO()
tree = ElementTree.ElementTree(elem)
tree.write(file, **options)
try:
encoding = options["encoding"]
except KeyError:
encoding = "utf-8"
result = fix_compatibility(file.getvalue().decode(encoding))
if sys.version_info[0] < 3:
result = result.encode(encoding)
return result
def render_plot(fig, format=None):
from matplotlib.backends.backend_agg import FigureCanvasAgg
if not fig or fig is None:
return ''
if format is None or not format:
format = 'png'
canvas = FigureCanvasAgg(fig)
buf = BytesIO()
canvas.print_figure(buf, format=format)
image_data = buf.getvalue()
return 'data:image/%s;base64,%s' % (quote(format),
quote(base64.b64encode(image_data)))
def histogram(series, **kwargs):
"""Plot an histogram of the data.
Parameters
----------
series: Series, default None
The data to plot.
Returns
-------
str, The resulting image encoded as a string.
"""
imgdata = BytesIO()
plot = _plot_histogram(series, **kwargs)
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
# TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
return result_string
def test_popen_io_readloop(monkeypatch, execmodel):
sio = BytesIO('test'.encode('ascii'))
io = Popen2IO(sio, sio, execmodel)
real_read = io._read
def newread(numbytes):
if numbytes > 1:
numbytes = numbytes-1
return real_read(numbytes)
io._read = newread
result = io.read(3)
assert result == 'tes'.encode('ascii')
def loads(data,
charset='utf-8',
errors=default_errors,
decode_strings=False,
object_hook=None,
array_hook=None):
"""Read a PHP-serialized object hierarchy from a string. Characters in the
string past the object's representation are ignored. On Python 3 the
string must be a bytestring.
"""
return load(BytesIO(data), charset, errors, decode_strings, object_hook,
array_hook)
def get(self, package):
self.set_header("Content-Type", "image/png")
period = self.get_argument('period', 'month')
url = PYPI_URL % package
downloads = self.intword(self.get_downloads(url, period))
period = "this_%s" % period if period in ('week', 'month') else "today"
pperiod = "%s_%s" % (downloads, period)
shield_url = SHIELD_URL % ("downloads", pperiod, 'brightgreen')
shield = requests.get(shield_url).content
img = BytesIO(shield)
img.seek(0)
self.write(img.read())
def mdl_1d(x, y):
"""builds univariate model to calculate AUC"""
lr = LogisticRegressionCV(scoring='roc_auc')
lars = LassoLarsIC(criterion='aic')
if x.nunique() > 10 and com.is_numeric_dtype(x):
x2 = sb_cutz(x)
series = pd.get_dummies(x2, dummy_na=True)
else:
series = pd.get_dummies(x, dummy_na=True)
lr.fit(series, y)
lars.fit(series, y)
try:
preds = (lr.predict_proba(series)[:, -1])
#preds = (preds > preds.mean()).astype(int)
except ValueError:
Tracer()()
# try:
# cm = confusion_matrix(y, (preds > y.mean()).astype(int))
# except ValueError:
# Tracer()()
aucz = roc_auc_score(y, preds)
ns = num_bin_stats(x, y)
nplot = plot_num(ns)
#plot = plot_confusion_matrix(cm, y)
imgdata = BytesIO()
nplot.savefig(imgdata)
imgdata.seek(0)
nplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdata.getvalue()))
plt.close()
bplot = plot_bubble(ns)
imgdatab = BytesIO()
bplot.savefig(imgdatab)
imgdatab.seek(0)
bplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdatab.getvalue()))
plt.close()
return aucz, nplot, bplot
def mini_histogram(histogram_data):
# Small histogram
imgdata = BytesIO()
hist_data = histogram_data
figure = plt.figure(figsize=(2, 0.75))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.axes.get_yaxis().set_visible(False)
plot.set_facecolor("w")
xticks = plot.xaxis.get_major_ticks()
for tick in xticks[1:-1]:
tick.set_visible(False)
tick.label.set_visible(False)
for tick in (xticks[0], xticks[-1]):
tick.label.set_fontsize(8)
plot.figure.subplots_adjust(left=0.15, right=0.85, top=1, bottom=0.35, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
result_string = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
plt.close(plot.figure)
return result_string
def save_poster():
buffer_image = BytesIO()
buffer_image.seek(0)
base64image = request.form['image']
name = request.form['name']
base64image = re.sub('data:image/png;base64,','',str(base64image))
base64image = re.sub('\n','',base64image)
poster_image = Image.open(BytesIO(base64.b64decode(base64image)))
poster_image.save(buffer_image, 'JPEG', quality=90)
buffer_image.seek(0)
redis.set(name, buffer_image.getvalue())
return json.dumps({'success':True}), 200, {'ContentType':'application/json'}
def send_fcgi_response(request, data, response):
io = BytesIO(data)
rec = Record()
rec.read(io)
resp = rec
resp.type = FCGI_STDOUT
resp.contentData = response
resp.contentLength = len(response)
out = BytesIO()
resp.write(out)
out.seek(0)
request.sendall(out.read())
resp.type = FCGI_END_REQUEST
resp.contentData = ""
resp.contentLength = 0
out = BytesIO()
resp.write(out)
out.seek(0)
request.sendall(out.read())
def occi_curl(base_url=None, url='/-/', authtype=None, ignoressl=None, user=None, passwd=None, mimetype=None, headers=[], post='', custom_request=''):
"""Send HTTP request
:param string base_url: OCCI server URL (default: from config)
:param string url: path element of the URL
:param string authtype: authentication type (default: from config)
:param bool ignoressl: ignore SSL problems (default: from config)
:param string user: user name for 'basic' auth (default: from config)
:param string passwd: password for 'basic' auth (default: from config)
:param string mimetype: accepted mimetype (empty string='\*/\*')
:param string headers[]: HTTP Headers
:param string post: HTTP Body
:param string custom_request: HTTP Request type (default: 'GET' or 'POST')
:return: [body, header, HTTP status, content type]
:rtype: [string[], string[], string, string]
"""
global header
if base_url is None:
base_url = occi_config['url']
if authtype is None:
authtype = occi_config['authtype']
if ignoressl is None:
ignoressl = occi_config['ignoressl']
if user is None:
user = occi_config['user']
if passwd is None:
passwd = occi_config['passwd']
if mimetype is None:
mimetype = occi_config['mimetype']
curlverbose = occi_config['curlverbose']
if sys.version_info >= (3,):
buffer = BytesIO()
else:
buffer = StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, str(base_url + url))
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
# Disable check of SSL certificate
if ignoressl:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
if 'capath' in occi_config and occi_config['capath']:
curl.setopt(pycurl.CAPATH, occi_config['capath'])
if 'cachain' in occi_config and occi_config['cachain']:
curl.setopt(pycurl.CAINFO, occi_config['cachain'])
# Name and password for basic auth (ONLY SUPPORTED YET)
if authtype == "basic":
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
curl.setopt(pycurl.USERPWD, "%s:%s" % (user, passwd))
elif authtype == "x509":
if 'cert' in occi_config and occi_config['cert']:
curl.setopt(pycurl.SSLCERT, occi_config['cert'])
if 'key' in occi_config and occi_config['key']:
curl.setopt(pycurl.SSLKEY, occi_config['key'])
if 'passphrase' in occi_config and occi_config['passphrase']:
curl.setopt(pycurl.SSLCERTPASSWD, occi_config['passphrase'])
# Verbose mode
curl.setopt(pycurl.VERBOSE, curlverbose)
curl.setopt(pycurl.CONNECTTIMEOUT, occi_config['connectiontimeout'])
curl.setopt(pycurl.TIMEOUT, occi_config['timeout'])
# Set appropriate mime type
if mimetype:
headers = ['Accept: %s' % mimetype] + headers
else:
headers = ['Accept: */*'] + headers
# Set requested HTTP headers
if headers:
curl.setopt(pycurl.HTTPHEADER, headers)
# HTTP header response
if sys.version_info >= (3,):
curl.setopt(pycurl.HEADERFUNCTION, get_header3)
else:
curl.setopt(pycurl.HEADERFUNCTION, get_header2)
if post or custom_request == 'POST':
curl.setopt(pycurl.POST, 1)
if post:
curl.setopt(pycurl.POSTFIELDS, post)
else:
curl.setopt(pycurl.POSTFIELDS, 'OK')
if curlverbose:
print "==== POST ==== "
print post
print "============== "
if custom_request and custom_request != 'POST':
curl.setopt(pycurl.CUSTOMREQUEST, custom_request)
# DO IT!
header = []
try:
curl.perform()
curl.close()
except pycurl.error as pe:
raise occi.Error(pe)
## 'Server: Apache/2.2.22 (Debian)\r\n'
h = {}
for item in header:
if re.match(r'.*:.*', item):
key = re.sub(r':.*', r'', item.rstrip())
value = re.sub(r'([^:]*):\s*(.*)', r'\2', item.rstrip())
h[key] = value
else:
if re.match(r'^HTTP', item):
http_status = item.rstrip()
content_type = None
if 'Content-Type' in h:
content_type = re.split(';', h['Content-Type'])[0]
body = buffer.getvalue()
if sys.version_info >= (3,):
encoding = 'iso-8859-1'
if content_type:
match = re.search(r';\s*charset=(\S+)', h['Content-Type'])
if match:
encoding = match.group(1)
body = body.decode(encoding)
return [body.splitlines(), header, http_status, content_type]
def describe_float_1d(df, column, current_result, nrows):
if spark_version == "1.6+":
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
variance(col(column)).alias("variance"),
kurtosis(col(column)).alias("kurtosis"),
stddev(col(column)).alias("std"),
skewness(col(column)).alias("skewness"),
df_sum(col(column)).alias("sum")
).toPandas()
else:
stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"),
df_min(col(column)).alias("min"),
df_max(col(column)).alias("max"),
df_sum(col(column)).alias("sum")
).toPandas()
stats_df["variance"] = df.select(column).na.drop().agg(variance_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
stats_df["std"] = np.sqrt(stats_df["variance"])
stats_df["skewness"] = df.select(column).na.drop().agg(skewness_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
stats_df["kurtosis"] = df.select(column).na.drop().agg(kurtosis_custom(col(column),
stats_df["mean"].ix[0],
current_result["count"])).toPandas().ix[0][0]
for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]):
stats_df[pretty_name(x)] = (df.select(column)
.na.drop()
.selectExpr("percentile_approx(`{col}`,CAST({n} AS DOUBLE))"
.format(col=column, n=x)).toPandas().ix[:,0]
)
stats = stats_df.ix[0].copy()
stats.name = column
stats["range"] = stats["max"] - stats["min"]
stats["iqr"] = stats[pretty_name(0.75)] - stats[pretty_name(0.25)]
stats["cv"] = stats["std"] / float(stats["mean"])
stats["mad"] = (df.select(column)
.na.drop()
.select(df_abs(col(column)-stats["mean"]).alias("delta"))
.agg(df_sum(col("delta"))).toPandas().ix[0,0] / float(current_result["count"]))
stats["type"] = "NUM"
stats['n_zeros'] = df.select(column).where(col(column)==0.0).count()
stats['p_zeros'] = stats['n_zeros'] / float(nrows)
# Large histogram
imgdata = BytesIO()
hist_data = create_hist_data(df, column, stats["min"], stats["max"], bins)
figure = plt.figure(figsize=(6, 4))
plot = plt.subplot()
plt.bar(hist_data["left_edge"],
hist_data["count"],
width=hist_data["width"],
facecolor='#337ab7')
plot.set_ylabel("Frequency")
plot.figure.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.1, wspace=0, hspace=0)
plot.figure.savefig(imgdata)
imgdata.seek(0)
stats['histogram'] = 'data:image/png;base64,' + quote(base64.b64encode(imgdata.getvalue()))
#TODO Think about writing this to disk instead of caching them in strings
plt.close(plot.figure)
stats['mini_histogram'] = mini_histogram(hist_data)
return stats