def show(self):
r"""This will send any existing plots to the IPython notebook.
If yt is being run from within an IPython session, and it is able to
determine this, this function will send any existing plots to the
notebook for display.
If yt can't determine if it's inside an IPython session, it will raise
YTNotInsideNotebook.
Examples
--------
>>> import yt
>>> ds = yt.load('IsolatedGalaxy/galaxy0030/galaxy0030')
>>> pp = ProfilePlot(ds.all_data(), 'density', 'temperature')
>>> pp.show()
"""
if "__IPYTHON__" in dir(builtins):
api_version = get_ipython_api_version()
if api_version in ('0.10', '0.11'):
self._send_zmq()
else:
from IPython.display import display
display(self)
else:
raise YTNotInsideNotebook
def _ipython_display_(self, **kwargs):
"""Called when `IPython.display.display` is called on the widget."""
def loud_error(message):
self.log.warn(message)
sys.stderr.write('%s\n' % message)
# Show view.
if self._view_name is not None:
validated = Widget._version_validated
# Before the user tries to display a widget. Validate that the
# widget front-end is what is expected.
if validated is None:
loud_error('Widget Javascript not detected. It may not be installed properly.')
elif not validated:
loud_error('The installed widget Javascript is the wrong version.')
# TODO: delete this sending of a comm message when the display statement
# below works. Then add a 'text/plain' mimetype to the dictionary below.
self._send({"method": "display"})
# The 'application/vnd.jupyter.widget' mimetype has not been registered yet.
# See the registration process and naming convention at
# http://tools.ietf.org/html/rfc6838
# and the currently registered mimetypes at
# http://www.iana.org/assignments/media-types/media-types.xhtml.
# We don't have a 'text/plain' entry so that the display message will be
# will be invisible in the current notebook.
data = { 'application/vnd.jupyter.widget': self._model_id }
display(data, raw=True)
self._handle_displayed(**kwargs)
def draw_strokes(data, factor=10, svg_filename = 'sample.svg'):
min_x, max_x, min_y, max_y = get_bounds(data, factor)
dims = (50 + max_x - min_x, 50 + max_y - min_y)
dwg = svgwrite.Drawing(svg_filename, size=dims)
dwg.add(dwg.rect(insert=(0, 0), size=dims,fill='white'))
lift_pen = 1
abs_x = 25 - min_x
abs_y = 25 - min_y
p = "M%s,%s " % (abs_x, abs_y)
command = "m"
for i in xrange(len(data)):
if (lift_pen == 1):
command = "m"
elif (command != "l"):
command = "l"
else:
command = ""
x = float(data[i,0])/factor
y = float(data[i,1])/factor
lift_pen = data[i, 2]
p += command+str(x)+","+str(y)+" "
the_color = "black"
stroke_width = 1
dwg.add(dwg.path(p).stroke(the_color,stroke_width).fill("none"))
dwg.save()
display(SVG(dwg.tostring()))
def remove_row(self, value=None):
"""Remove the current row from the table"""
if self._multi_index:
msg = "Cannot remove a row from a table with a multi index"
display(Javascript('alert("%s")' % msg))
return
self.send({'type': 'remove_row'})
def display(self):
"""
Display and format this widget.
"""
from IPython.display import display
display(self)
self.format()
def update(self, *args):
"""
Call the interact function and update the output widget with
the result of the function call.
Parameters
----------
*args : ignored
Required for this method to be used as traitlets callback.
"""
self.kwargs = {}
if self.manual:
self.manual_button.disabled = True
try:
show_inline_matplotlib_plots()
with self.out:
if self.clear_output:
clear_output(wait=True)
for widget in self.kwargs_widgets:
value = widget.get_interact_value()
self.kwargs[widget._kwarg] = value
self.result = self.f(**self.kwargs)
show_inline_matplotlib_plots()
if self.auto_display and self.result is not None:
display(self.result)
except Exception as e:
ip = get_ipython()
if ip is None:
self.log.warn("Exception in interact callback: %s", e, exc_info=True)
else:
ip.showtraceback()
finally:
if self.manual:
self.manual_button.disabled = False
def runVerification():
CALIBRATION_DIR = PARENT_DIR/'05 Specimen Images'/'01 Calibration Images'/'verification'
verif = DataTree()
verif.files = DataTree(**{
'front':'cropped-DSC00600.png',
'front21':'cropped-DSC00601.png',
'front22':'cropped-DSC00602.png',
'front23':'cropped-DSC00603.png',
'side':'cropped-DSC00598.png',
'front_eraser':'cropped-DSC00594.png',
'front_eraser_rot':'cropped-DSC00594 2.png',
'side_eraser':'cropped-DSC00596.png',
})
verif.files = {k: (CALIBRATION_DIR/n).resolve() for k,n in verif.files.items() }
verif.files
dims = DIMS['lg']
pd = verif.files['front'].parent / 'processed'
pixel_ratio = DIMS['pixel_ratio']
pd
for f,tb in [('front21',(100+150,300+550)), ('front22',(100+150,300+550))]:
debug("main",f, tb)
locations = DataTree(front=DataTree(top=DataTree(y=tb[0]),bottom=DataTree(y=tb[1])))
res,fig = process(pd, verif.files[f], DIMS, 'lg', 'front',locations=locations)
display(Image(fig))
def print_table(table, name=None, fmt=None):
"""Pretty print a pandas DataFrame.
Uses HTML output if running inside Jupyter Notebook, otherwise
formatted text output.
Parameters
----------
table : pandas.Series or pandas.DataFrame
Table to pretty-print.
name : str, optional
Table name to display in upper left corner.
fmt : str, optional
Formatter to use for displaying table elements.
E.g. '{0:.2f}%' for displaying 100 as '100.00%'.
Restores original setting after displaying.
"""
if isinstance(table, pd.Series):
table = pd.DataFrame(table)
if fmt is not None:
prev_option = pd.get_option('display.float_format')
pd.set_option('display.float_format', lambda x: fmt.format(x))
if name is not None:
table.columns.name = name
display(table)
if fmt is not None:
pd.set_option('display.float_format', prev_option)
def _update(self, p):
if p <= self.size:
js = Javascript("jQuery('#%s').val('%i')" % (self.id, p))
display(js)
self.progress = p
else:
raise RuntimeError("Already reached 100%")
def loadFromDb(identificador):
import sys
if not sys.version_info[:2] == (3, 4):
print ('Sos un boludo!, pero uno previsor')
print ('Este codigo esta pensado para correr en python 3.4')
import os
import pandas as pd
from IPython.display import display
display ('Verificando datos guardados previamente')
filenameTouchs = 'dbTouchs'
filenameSounds = 'dbSounds'
if os.path.isfile(filenameTouchs):
touchsLoad = pd.read_pickle(filenameTouchs)
display ('Datos de touchs previos cargados')
else:
display ('Datos de touchs previos inexistentes')
touchsLoad = pd.DataFrame()
if os.path.isfile(filenameSounds):
soundsLoad = pd.read_pickle(filenameSounds)
display ('Datos de sounds previos cargados')
else:
display ('Datos de sounds previos inexistentes')
soundsLoad = pd.DataFrame()
if identificador!=0:
touchsLoad = touchsLoad[touchsLoad['identificador']==identificador]
soundsLoad = soundsLoad[soundsLoad['identificador']==identificador]
return touchsLoad, soundsLoad
def _compute_current_density(bvs, gvx, gvy, gvz, cmatr, cmati, occvec, verbose=True):
"""Compute the current density in each cartesian direction."""
nbas, npts = bvs.shape
curx = np.zeros(npts, dtype=np.float64)
cury = np.zeros(npts, dtype=np.float64)
curz = np.zeros(npts, dtype=np.float64)
cval = np.zeros(nbas, dtype=np.float64)
if verbose:
fp = FloatProgress(description='Computing:')
display(fp)
for mu in range(nbas):
if verbose:
fp.value = mu / nbas * 100
crmu = cmatr[mu]
cimu = cmati[mu]
bvmu = bvs[mu]
gvxmu = gvx[mu]
gvymu = gvy[mu]
gvzmu = gvz[mu]
for nu in range(nbas):
crnu = cmatr[nu]
cinu = cmati[nu]
bvnu = bvs[nu]
gvxnu = gvx[nu]
gvynu = gvy[nu]
gvznu = gvz[nu]
cval = evaluate('-0.5 * (occvec * (crmu * cinu - cimu * crnu))', out=cval)
csum = cval.sum()
evaluate('curx + csum * (bvmu * gvxnu - gvxmu * bvnu)', out=curx)
evaluate('cury + csum * (bvmu * gvynu - gvymu * bvnu)', out=cury)
evaluate('curz + csum * (bvmu * gvznu - gvzmu * bvnu)', out=curz)
if verbose:
fp.close()
return curx, cury, curz
def enable_ipython(self, **kwargs):
"""
Enable plotting in the iPython notebook.
Once enabled, all lightning plots will be automatically produced
within the iPython notebook. They will also be available on
your lightning server within the current session.
"""
# inspired by code powering similar functionality in mpld3
# https://github.com/jakevdp/mpld3/blob/master/mpld3/_display.py#L357
from IPython.core.getipython import get_ipython
from IPython.display import display, Javascript, HTML
self.ipython_enabled = True
self.set_size('medium')
ip = get_ipython()
formatter = ip.display_formatter.formatters['text/html']
if self.local_enabled:
from lightning.visualization import VisualizationLocal
js = VisualizationLocal.load_embed()
display(HTML("<script>" + js + "</script>"))
if not self.quiet:
print('Running local mode, some functionality limited.\n')
formatter.for_type(VisualizationLocal, lambda viz, kwds=kwargs: viz.get_html())
else:
formatter.for_type(Visualization, lambda viz, kwds=kwargs: viz.get_html())
r = requests.get(self.get_ipython_markup_link(), auth=self.auth)
display(Javascript(r.text))
def finish(self):
"""Puts the ProgressBar bar in the finished state."""
self.finished = True
self.update(self.maxval)
self.start_time = None
# Clean up notebook stuff, quite differently from regular
if not _is_ipython_notebook():
self.fd.write('\n')
else:
from IPython.display import display, Javascript
# First delete the node that held the progress bar from the page
js = """var element = document.getElementById('%s');
var parent = element.parentNode
parent.removeChild(element);
parent.parentElement.remove();""" % self.uuid
display(Javascript(js))
# Then also remove its trace from the cell output (so it doesn't get
# stored with the notebook). This needs to be done for all widgets as
# well as for progressBar
uuids = [str(self.uuid)]
uuids += [w.uuid for w in self.widgets if isinstance(w, widgets.Widget)]
display(Javascript('this.cleanProgressBar(%s);' % uuids))
# Save the finished line so we can see how much time it took
self.fd.write(self._format_line() + '\n')
self.fd.flush()
if self.signal_set:
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
def ishow(cls, figure):
""" Display a static image of the plot described by `figure`
in an IPython Notebook.
"""
img = cls.get(figure)
from IPython.display import display, Image
display(Image(img))
def interact(__interact_f=None, **kwargs):
"""interact(f, **kwargs)
Interact with a function using widgets."""
# positional arg support in: https://gist.github.com/8851331
if __interact_f is not None:
# This branch handles the cases:
# 1. interact(f, **kwargs)
# 2. @interact
# def f(*args, **kwargs):
# ...
f = __interact_f
w = interactive(f, **kwargs)
f.widget = w
display(w)
return f
else:
# This branch handles the case:
# @interact(a=30, b=40)
# def f(*args, **kwargs):
# ...
def dec(f):
w = interactive(f, **kwargs)
f.widget = w
display(w)
return f
return dec
def display_img_array(ima, **kwargs):
if ima.dtype == np.float32 or ima.dtype == np.float64:
ima = (np.clip(ima, 0., 1.)*255).astype(np.uint8)
im = PIL_Image.fromarray(ima)
bio = BytesIO()
im.save(bio, format='png')
display(Image(bio.getvalue(), format='png', **kwargs))
def pretty_notebook(line):
display(HTML("""
<style>
/* narrow output lines */
.text_cell_render p {
width: 105ex;
}
.text_cell_render {
font-family: Helvetica Neue;
font-size: 17px;
}
.text_cell_render code,
.text_cell_render kbd,
.text_cell_render pre,
.text_cell_render samp,
.CodeMirror,
.output_text pre {
font-family: Inconsolata, monospace;
font-size: 17px;
}
h1, h2 {
color: #5F452C;
}
h1 {
text-decoration: underline;
}
h3, h4 {
color: #333333;
}
h4 {
font-style: italic;
}
</style>
"""))
def _update_ipython_widget(self, value=None):
"""
Update the progress bar to the given value (out of a total
given to the constructor).
This method is for use in the IPython notebook 2+.
"""
# Create and display an empty progress bar widget,
# if none exists.
if not hasattr(self, '_widget'):
# Import only if an IPython widget, i.e., widget in iPython NB
if ipython_major_version < 4:
from IPython.html import widgets
self._widget = widgets.FloatProgressWidget()
else:
from ipywidgets import widgets
self._widget = widgets.FloatProgress()
from IPython.display import display
display(self._widget)
self._widget.value = 0
# Calculate percent completion, and update progress bar
frac = (value/self._total)
self._widget.value = frac * 100
self._widget.description =' ({:>6.2%})'.format(frac)
def get_and_show():
env.host = '180.150.190.52:6'
env.user = "******"
while True:
clear_output()
remote_filename = REMOTE_PATH+'/'+'ocr_20150902.log'
#local_filename = LOCAL_PATH +'_logs/'+'audit_logSessionServer.txt' + '.'+env.host
remote_ret = run(""" tail -n10 %s | grep -E 'Downloading http|Return result' """ % remote_filename)
lines = remote_ret.split('\n');
for line in lines:
items = line.split()
#print items
if len(items)>=13 and items[11]=="Downloading":
process_id = items[10]
img_url = items[12]
#print img_url
if (process_id in records) and records[process_id] != img_url:
print process_id,":", "Exception! does't complete it work."
display(Image(url=records[process_id]))
records.pop(process_id);
records[process_id] = img_url
continue
if len(items)>=15 and items[12]=="result:":
process_id = items[10][:-1]
ocr_result = " ".join(items[14:-1])
if (process_id in records):
print process_id,":", ocr_result
display(Image(url=records[process_id]))
records.pop(process_id);
continue
print "processing:", records
print "b"
time.sleep(2)
def init_notebook_mode():
style_inject = """
<style>
div.htmresearchviz-output {
-webkit-touch-callout: none;
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
padding-bottom: 2px;
}
div.htmresearchviz-output svg {
max-width: initial;
}
</style>
"""
script_inject = u"""
<script type='text/javascript'>
if(!window.htmresearchviz0) {{
define('htmresearchviz0', function(require, exports, module) {{
{script}
}});
require(['htmresearchviz0'], function(htmresearchviz0) {{
window.htmresearchviz0 = htmresearchviz0;
}});
}}
</script>
""".format(script=get_htmresearchviz0_js())
display(HTML(style_inject + script_inject))
def bowtie(self, line=''):
"""Build and serve a Bowtie app."""
opts, appvar = self.parse_options(line, 'w:h:b:p:')
width = opts.get('w', 1500)
height = opts.get('h', 1000)
border = opts.get('b', 2)
port = opts.get('p', 9991)
host = '0.0.0.0'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((host, port))
if result == 0:
raise Exception(f'Port {port} is unavailable on host {host}, aborting.')
global_ns = self.shell.user_global_ns
local_ns = self.shell.user_ns
try:
# pylint: disable=eval-used
app = eval(appvar, global_ns, local_ns)
except NameError:
raise UsageError(f'Could not find App {appvar}')
if not isinstance(app, App):
raise UsageError(f'App is of type {type(app)} needs to be type <bowtie.App>')
app._build(notebook=get_notebook_name()) # pylint: disable=protected-access
self.process = Process(target=app._serve) # pylint: disable=protected-access
self.process.start()
time.sleep(5)
clear_output()
display(HTML(
f'<iframe src=http://localhost:9991 width={width} height={height} '
f'frameBorder={border}></iframe>'
))
def displayFileDialog(self):
self.inputFile = widgets.TextWidget(
description='Input file name (csv)')
button = widgets.ButtonWidget(description="Process file")
display(self.inputFile, button)
processCsvFileFn = lambda fileName: self.processCsvFile()
button.on_click(processCsvFileFn)
def button_click(clicked):
if ipython_less_than_3:
self.button.add_class('disabled')
else:
self.button._dom_classes = ['disabled']
self.button.description = 'Rerunning Simulation ...'
self.constant_values = []
for i in self._widget_dict.values():
self.constant_values.append(i.value)
if self.system is not None:
# update system constants
self.system.constants = dict(zip(self.system.constants,
self.constant_values))
self.generate_visualization_json_system(self.system)
else:
self.generate_visualization_json(
self.dynamic_variables,
self.constant_variables, self.dynamic_values,
self.constant_values,
fps=self.fps,
outfile_prefix=self.outfile_prefix)
self.create_static_html(overwrite=True, silent=True)
js = 'jQuery("#json-input").val("{}");'.format('static/' +
self.scene_json_file)
display(Javascript(js))
display(Javascript('jQuery("#simulation-load").click()'))
if ipython_less_than_3:
self.button.remove_class('disabled')
else:
self.button._dom_classes = ['enabled']
self.button.description = 'Rerun Simulation'
def play_sequence(sequence,
synth=midi_synth.synthesize,
sample_rate=_DEFAULT_SAMPLE_RATE,
colab_ephemeral=True,
**synth_args):
"""Creates an interactive player for a synthesized note sequence.
This function should only be called from a Jupyter or Colab notebook.
Args:
sequence: A music_pb2.NoteSequence to synthesize and play.
synth: A synthesis function that takes a sequence and sample rate as input.
sample_rate: The sample rate at which to synthesize.
colab_ephemeral: If set to True, the widget will be ephemeral in Colab, and
disappear on reload (and it won't be counted against realtime document
size).
**synth_args: Additional keyword arguments to pass to the synth function.
"""
array_of_floats = synth(sequence, sample_rate=sample_rate, **synth_args)
try:
import google.colab # pylint: disable=unused-import,unused-variable,g-import-not-at-top
colab_play(array_of_floats, sample_rate, colab_ephemeral)
except ImportError:
display.display(display.Audio(array_of_floats, rate=sample_rate))
def __init__(self, *args, **kwargs):
'''
This will be called each time an implementing class is instantiated
'''
# as a good citizen
extension = os.path.basename(self._view_static)
# copy the static files to a namespaced location in `nbextensions`
install_nbextension(os.path.abspath(self._view_static), verbose=0)
# this assumes your extension takes care of its own dependencies...
script = 'IPython.load_extensions("%s/%s");' % (
extension,
self._view_module
)
# again, assume that you have put the style in the extension's folder
styles = []
try:
styles = ['/nbextensions/%s/%s.css' % (extension, self._view_style)]
except:
pass
# tell the front-end to request the assets
display(Javascript(script, css=styles))
# always call the parent constructor!
super(InstallerMixin, self).__init__(*args, **kwargs)
def axes_set_aspect_ratio(ax, ratio = 'equal', show = True):
"""
function that accepts an Axes instance and update the information
setting the aspect ratio of the axis to the defined quantity
Parameters:
-----------
ax : matplotlib.axes.Axes or matplotlib.axes.Subplot instance
ratio : str or int/float
The value can be a string with the following values:
'equal' : (default) same scaling from data to plot units for x and y
'auto' : automatic; fill position rectangle with data
Or a:
number (int or float) : a circle will be stretched such that the
height is num times the width. aspec t =1 is the same as
aspect='equal'.
show : bool
if `True` the figure will be shown.
If you are working in a rich display environment like the IPython
qtconsole or the Jupyter notebook it will use
`IPython.display.display` to show the figure.
If you are working otherwise it will call the `show` of the
`Figure` instance.
"""
ax.set_aspect(ratio, adjustable = None)
if show:
if RICH_DISPLAY:
display(ax.figure)
else:
ax.figure.show()
def _show_entire_graph(graph_def, max_const_size=32):
"""Visualize TensorFlow graph.
Parameters
----------
graph_def : TYPE
Description
max_const_size : int, optional
Description
"""
if hasattr(graph_def, 'as_graph_def'):
graph_def = graph_def.as_graph_def()
strip_def = _strip_consts(graph_def, max_const_size=max_const_size)
code = """
<script>
function load() {{
document.getElementById("{id}").pbtxt = {data};
}}
</script>
<link rel="import" href="https://tensorboard.appspot.com/tf-graph-basic.build.html" onload=load()>
<div style="height:600px">
<tf-graph-basic id="{id}"></tf-graph-basic>
</div>
""".format(data=repr(str(strip_def)), id='graph' + str(np.random.rand()))
iframe = """
<iframe seamless style="width:800px;height:620px;border:0" srcdoc="{}"></iframe>
""".format(code.replace('"', '"'))
display(HTML(iframe))
def export(self, timestamp=None):
"""
Get the current notebook data and export.
"""
if self.skip_notebook_export:
super(NotebookArchive, self).export(timestamp=self._timestamp,
info={'notebook':self.notebook_name})
return
self.export_success = None
self._notebook_data = io.StringIO()
name = self.namespace
# Unfortunate javascript hacks to get at notebook data
capture_cmd = ((r"var capture = '%s._notebook_data.write(r\"\"\"'" % name)
+ r"+json_string+'\"\"\".decode(\'utf-8\'))'; ")
cmd = (r'var kernel = IPython.notebook.kernel; '
+ r'var json_data = IPython.notebook.toJSON(); '
+ r'var json_string = JSON.stringify(json_data); '
+ capture_cmd
+ "var pycmd = capture + ';%s._export_with_html()'; " % name
+ r"kernel.execute(pycmd)")
tstamp = time.strftime(self.timestamp_format, self._timestamp)
export_name = self._format(self.export_name, {'timestamp':tstamp, 'notebook':self.notebook_name})
print(('Export name: %r\nDirectory %r' % (export_name,
os.path.join(os.path.abspath(self.root))))
+ '\n\nIf no output appears, please check holoviews.archive.last_export_status()')
display(Javascript(cmd))
def update_perceptron_plot(h, f, ax, x_plus, x_minus, i, w, b, x_select):
"""Update plots after decision boundary has changed."""
# Helper function for updating plots
h["select"].set_xdata(x_select[0])
h["select"].set_ydata(x_select[1])
# Re-plot the hyper plane
plot_limits = {}
plot_limits["x"] = np.asarray(ax[0].get_xlim())
plot_limits["y"] = np.asarray(ax[0].get_ylim())
x0, x1 = hyperplane_coordinates(w, b, plot_limits)
strt = -b / w[1]
h["arrow"].remove()
del (h["arrow"])
h["arrow"] = ax[0].arrow(0, strt, w[0], w[1] + strt, head_width=0.2)
h["plane"].set_xdata(x0)
h["plane"].set_ydata(x1)
h["iter"].set_text("Update " + str(i))
ax[1].cla()
bins = 15
f_minus = np.dot(x_minus, w)
f_plus = np.dot(x_plus, w)
ax[1].hist(f_plus, bins, alpha=0.5, label="+1", color="r")
ax[1].hist(f_minus, bins, alpha=0.5, label="-1", color="g")
ax[1].legend(loc="upper right")
display(f)
clear_output(wait=True)
if i < 3:
time.sleep(0.5)
else:
time.sleep(0.25)
return h
def code_toggle(start_show=False, message=None):
"""Toggling on and off code in a notebook.
:param start_show: Whether to display the code or not on first load (default is False).
:type start_show: bool
:param message: the message used to toggle display of the code.
:type message: string
The tip that this idea is
based on is from Damian Kao (http://blog.nextgenetics.net/?e=102)."""
html ='<script>\n'
if message is None:
message = u'The raw code for this jupyter notebook can be hidden for easier reading.'
if start_show:
html += u'code_show=true;\n'
else:
html += u'code_show=false;\n'
html+='''function code_toggle() {
if (code_show){
$('div.input').show();
} else {
$('div.input').hide();
}
code_show = !code_show
}
$( document ).ready(code_toggle);
</script>
'''
html += message + ' To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.'
display(HTML(html))
def render_ipython(node):
"""Convenience method for rendering AST node in IPython environment."""
from IPython.display import display, Code # pylint: disable=import-error
display(Code(render_module(node), language='python'))
def display_javascript(widget, js_text):
# This will not work if javascript is disabled.
return display(Javascript(data=js_text))
activity_vectors_new = np.concatenate((targets, activity_vectors[:,:-1].T), axis=0)
gini_gain = GiniGain(activity_vectors_new)
activity_vectors_new = activity_vectors[:,:-1]
args_sorted = gini_gain.argsort()
out = activity_vectors_new[:,args_sorted]
else:
raise NotImplementedError("Unknown state space transform!")
return np.concatenate([out, activity_vectors[:,-1, None]], axis=1)
def L1(out, target):
return np.abs(out - target)
if __name__ == '__main__':
dataset_df = get_table()
display(dataset_df)
# get the name of the packages that is running in the system
text = 'packages_running_'
running_packages = [i for i in dataset_df.columns if text in i]
# creating the activity vector containing info about running apps and battery_plugged
extra_columns = ['battery_plugged']
activity_vectors_df = dataset_df[[*extra_columns, *running_packages]]
activity_vectors = activity_vectors_df.dropna().to_numpy()
# battery usage for each activity vector
battery_usage = dataset_df['battery_level'].to_list()
print(battery_usage)
# convert to a sequence of labels, note that nan was dropped above
def showarray(a, fmt='jpeg'):
a = np.uint8(np.clip(a, 0, 255))
f = StringIO()
PIL.Image.fromarray(a).save(f, fmt)
display(Image(data=f.getvalue()))
def _display(html):
return display(HTML(html))
# Create Reports ---------------------------------------------------------------
d = {
'Forward Bearing': round(forwardInitialBearing, 4),
'Backward Bearing': round(backwardInitialBearing, 4),
'Distance': round(cumulativeOffset, 4)
}
html = "<b> Linear Misclosure</b>"
html += """<table border="1">
<tr><th>Pass 1 Bearing [degrees]</th><th>Pass 2 Bearing [degrees] </th><th>Cumulative Offset [m]</th></tr>"""
html += "<tr><td>{}</td>".format(d['Forward Bearing'])
html += "<td>{}</td>".format(d['Backward Bearing'])
html += "<td>{}</td>".format(d['Distance'])
html += "</tr></table>"
html += "<small><small> Bearings are relative to True North </small></small>"
display(HTML(html))
# OOS Verification
d = {
'Run': pd.Series([run for run in runs]),
'KP From': pd.Series(["{0:.4f}".format(kpFrom) for kpFrom in kpFroms]),
'KP To': pd.Series([round(kpTo, 4) for kpTo in kpTos]),
'HOOS Sigma': pd.Series([round(hoosSigma, 3) for hoosSigma in hoosSigmas]),
'HOOS 2Sigma':
pd.Series([round(hoos2Sigma, 3) for hoos2Sigma in hoos2Sigmas]),
'VOOS Sigma': pd.Series([round(voosSigma, 3) for voosSigma in voosSigmas]),
'VOOS 2Sigma':
pd.Series([round(voos2Sigma, 3) for voos2Sigma in voos2Sigmas])
}
df = pd.DataFrame(d)
def OMP_GS(imDims, sparsity, measurements, A, MIDCT):
numPixels = imDims[0] * imDims[1]
r = measurements.copy()
r = r.reshape(len(measurements))
indices = []
bases = []
projection = np.zeros(len(measurements))
# Threshold to check error. If error is below this value, stop.
THRESHOLD = 0.01
# For iterating to recover all signals
i = 0
b = r.copy()
while i < sparsity and np.linalg.norm(r) > THRESHOLD:
# Calculate the correlations
print('%d - ' % i, end="", flush=True)
corrs = A.T.dot(r)
# Choose highest-correlated pixel location and add to collection
best_index = np.argmax(np.abs(corrs))
indices.append(best_index)
# Gram-Schmidt Method
if i == 0:
bases = A[:, best_index] / np.linalg.norm(A[:, best_index], 2)
else:
bases = Gram_Schmidt_rec(bases, A[:, best_index])
if i == 0:
newsubspace = bases
else:
newsubspace = bases[:, i]
projection = projection + (newsubspace.T.dot(b)) * newsubspace
# Find component parallel to subspace to use for next measurement
r = b - projection
Atrunc = A[:, indices]
if i % 20 == 1 or i == sparsity - 1 or np.linalg.norm(r) <= THRESHOLD:
xhat = np.linalg.lstsq(Atrunc, b)[0]
# `recovered_signal` = x
recovered_signal = np.zeros(numPixels)
for j, x in zip(indices, xhat):
recovered_signal[j] = x
ihat = np.dot(MIDCT, recovered_signal).reshape(imDims)
plt.imshow(ihat, cmap="gray", interpolation='nearest')
plt.title('Estimated Image from %s Measurements' %
str(len(measurements)))
display.clear_output(wait=True)
display.display(plt.gcf())
i = i + 1
display.clear_output(wait=True)
return recovered_signal, ihat
def display_visual(graph_data, user_input, algorithm=None, problem=None):
initial_node_colors = graph_data['node_colors']
if user_input == False:
def slider_callback(iteration):
# don't show graph for the first time running the cell calling this function
try:
show_map(graph_data, node_colors=all_node_colors[iteration])
except:
pass
def visualize_callback(Visualize):
if Visualize is True:
button.value = False
global all_node_colors
iterations, all_node_colors, node = algorithm(problem)
solution = node.solution()
all_node_colors.append(final_path_colors(all_node_colors[0], problem, solution))
slider.max = len(all_node_colors) - 1
for i in range(slider.max + 1):
slider.value = i
#time.sleep(.5)
slider = widgets.IntSlider(min=0, max=1, step=1, value=0)
slider_visual = widgets.interactive(slider_callback, iteration=slider)
display(slider_visual)
button = widgets.ToggleButton(value=False)
button_visual = widgets.interactive(visualize_callback, Visualize=button)
display(button_visual)
if user_input == True:
node_colors = dict(initial_node_colors)
if isinstance(algorithm, dict):
assert set(algorithm.keys()).issubset({"Breadth First Tree Search",
"Depth First Tree Search",
"Breadth First Search",
"Depth First Graph Search",
"Uniform Cost Search",
"A-star Search"})
algo_dropdown = widgets.Dropdown(description="Search algorithm: ",
options=sorted(list(algorithm.keys())),
value="Breadth First Tree Search")
display(algo_dropdown)
elif algorithm is None:
print("No algorithm to run.")
return 0
def slider_callback(iteration):
# don't show graph for the first time running the cell calling this function
try:
show_map(graph_data, node_colors=all_node_colors[iteration])
except:
pass
def visualize_callback(Visualize):
if Visualize is True:
button.value = False
problem = GraphProblem(start_dropdown.value, end_dropdown.value, romania_map)
global all_node_colors
user_algorithm = algorithm[algo_dropdown.value]
iterations, all_node_colors, node = user_algorithm(problem)
solution = node.solution()
all_node_colors.append(final_path_colors(all_node_colors[0], problem, solution))
slider.max = len(all_node_colors) - 1
for i in range(slider.max + 1):
slider.value = i
#time.sleep(.5)
start_dropdown = widgets.Dropdown(description="Start city: ",
options=sorted(list(node_colors.keys())), value="Arad")
display(start_dropdown)
end_dropdown = widgets.Dropdown(description="Goal city: ",
options=sorted(list(node_colors.keys())), value="Fagaras")
display(end_dropdown)
button = widgets.ToggleButton(value=False)
button_visual = widgets.interactive(visualize_callback, Visualize=button)
display(button_visual)
slider = widgets.IntSlider(min=0, max=1, step=1, value=0)
slider_visual = widgets.interactive(slider_callback, iteration=slider)
display(slider_visual)
def __init__(self):
self.dataset = datastructure.DataSet()
# the 'out' construction is to refresh the text output at each update
# to stop text building up in the widget display
self.out = Output()
self.out_top = Output()
# Initialise
self.current_view = 'Time'
self.N_frames = 1
self.overlap = 0.5
# BUTTONS
items_load = ['Load Data', 'Reset (deletes loaded data)']
items_view = ['View Time', 'View FFT', 'View TF']
items_axes = ['xmin', 'xmax', 'ymin', 'ymax']
items_save = ['Save Dataset', 'Save Figure']
items_legend = ['Left', 'on/off', 'Right']
self.buttons_load = [
Button(description=i, layout=Layout(width='50%'))
for i in items_load
]
self.buttons_load[0].button_style = 'primary'
self.buttons_load[0].style.font_weight = 'bold'
self.buttons_load[1].button_style = 'warning'
self.buttons_load[1].style.font_weight = 'bold'
self.buttons_view = [
Button(description=i, layout=Layout(width='99%'))
for i in items_view
]
self.buttons_view[0].button_style = 'info'
self.buttons_view[1].button_style = 'info'
self.buttons_view[2].button_style = 'info'
self.buttons_match = Button(description='Match Amplitudes',
layout=Layout(width='99%'))
self.buttons_match.button_style = 'info'
self.buttons_save = [
Button(description=i, layout=Layout(width='50%'))
for i in items_save
]
self.buttons_save[0].button_style = 'success'
self.buttons_save[0].style.font_weight = 'bold'
self.buttons_save[1].button_style = 'success'
self.buttons_save[1].style.font_weight = 'bold'
self.button_X = Button(description='Auto X',
layout=Layout(width='10%'))
self.button_Y = Button(description='Auto Y',
layout=Layout(width='10%'))
self.button_X.button_style = 'success'
self.button_Y.button_style = 'success'
self.buttons_legend = [
Button(description=i, layout=Layout(width='16%'))
for i in items_legend
]
self.buttons_legend_toggle = ToggleButton(description='Click-and-drag',
layout=Layout(
width='16%',
alignitems='start'))
# TEXT/LABELS/DROPDOWNS
self.item_axis_label = Label(value="Adjust axes manually:",
layout=Layout(width='16%'))
self.text_axes = [
FloatText(value=0, description=i, layout=Layout(width='16%'))
for i in items_axes
]
self.text_axes = [self.button_X] + [self.button_Y] + [
self.item_axis_label
] + self.text_axes
self.item_legend_label = Label(value="Legend position:",
layout=Layout(width='16%'))
self.item_blank_label = Label(value="", layout=Layout(width='20%'))
group1 = VBox([self.buttons_view[0]], layout=Layout(width='33%'))
group2 = VBox([self.buttons_view[1]], layout=Layout(width='33%'))
group3 = VBox([self.buttons_view[2], self.buttons_match],
layout=Layout(width='33%'))
# ASSEMBLE
display(HBox(self.buttons_load))
self.p = plotting.PlotData()
display(HBox(self.text_axes))
display(
HBox([self.item_blank_label, self.item_legend_label] +
self.buttons_legend + [self.buttons_legend_toggle]))
display(HBox([group1, group2, group3]))
display(HBox(self.buttons_save))
# Make interactive
self.text_axes[3].observe(self.xmin, names='value')
self.text_axes[4].observe(self.xmax, names='value')
self.text_axes[5].observe(self.ymin, names='value')
self.text_axes[6].observe(self.ymax, names='value')
self.button_X.on_click(self.auto_x)
self.button_Y.on_click(self.auto_y)
self.buttons_legend[0].on_click(self.legend_left)
self.buttons_legend[1].on_click(self.legend_onoff)
self.buttons_legend[2].on_click(self.legend_right)
self.buttons_legend_toggle.observe(self.legend_toggle)
self.buttons_load[0].on_click(self.load_data)
self.buttons_load[1].on_click(self.undo)
self.buttons_view[0].on_click(self.time)
self.buttons_view[1].on_click(self.fft)
self.buttons_view[2].on_click(self.tf)
self.buttons_match.on_click(self.match)
self.buttons_save[0].on_click(self.save_data)
self.buttons_save[1].on_click(self.save_fig)
self.refresh_buttons()
# Put output text at bottom
display(self.out)
def view_id(self, item_id):
card = CardQuiz(item_id, self.data[str(item_id)])
display(card)
display(card.show())
def view_scope(self, with_cache=False, binary="", stats=[]):
bool_with_cache = with_cache
data = {
"arch": "X86",
"cpu": "Simple",
"clk": 1.0,
"size_l1": 16,
"assoc_l1": 2,
"latency_l1": 16,
"size_l2": 256,
"assoc_l2": 8,
"latency_l2": 20,
"memory": 'DDR3_1600',
"binary": binary,
"stats": stats
}
def on_button_clicked(b):
if b.name == 'simulate':
b.button_style = 'danger'
b.description = 'wait'
#try:
import sys
sys.path.insert(0, '.')
if bool_with_cache:
from cad4u.gem5.examples.two_level import cache_gem5
cache_gem5(data)
else:
from cad4u.gem5.examples.simple import simple_gem5
simple_gem5(data)
arguments = [
"sh", "/content/cad4u/gem5/execute.sh", data['arch'],
'/content/gem5_code.py'
]
self.execution(arguments)
self.output_gem5(data)
#except:
#print("erro!")
b.button_style = 'success'
b.description = "Start Simulate"
def on_value_change(change):
if change['owner'].name in data:
data[change['owner'].name] = change['new']
def create_Text(description="", button_style=""):
return Button(description=description,
button_style=button_style,
layout=Layout(height='auto', width='auto'))
def create_expanded_button(id,
description="",
button_style="",
disabled=False):
btn = Button(description=description,
button_style=button_style,
layout=Layout(height='auto', width='auto'),
disabled=disabled)
btn.name = id
btn.on_click(on_button_clicked)
return btn
def create_Float(id,
description="",
button_style="",
min=1,
max=10,
value=1,
step=1):
floatText = BoundedFloatText(description=description,
button_style=button_style,
layout=Layout(height='auto',
width='auto'),
min=min,
max=max,
value=value,
step=step)
floatText.name = id
floatText.observe(on_value_change, names='value')
return floatText
def create_Dropdown(id,
description="",
disabled=False,
options=[],
value='1'):
dropdown = Dropdown(description=description,
layout=Layout(height='30px', width='auto'),
value=value,
options=options,
disabled=disabled)
dropdown.name = id
dropdown.observe(on_value_change, names='value')
return dropdown
# create a 10x2 grid layout
grid = GridspecLayout(2, 10)
# fill it in with widgets
grid[0, 0] = create_Text("Architecture", "warning")
grid[0, 1] = create_Dropdown("arch",
options=["X86", "RISCV", "ARM"],
value="X86")
grid[0, 2] = create_Text("CPU", "warning")
grid[0,
3] = create_Dropdown("cpu",
options=["Simple", "In Order", "Out Order"],
value="Simple")
grid[1, 0] = create_Text("Clock (GHz)", "warning")
grid[1, 1] = create_Float("clk", value=1.0, min=0.2, max=5.0, step=0.1)
grid[1, 2] = create_Text("Memory", "warning")
grid[1, 3] = create_Dropdown("memory",
options=['DDR3_1600', 'DDR4_2400'],
value='DDR3_1600')
opts = []
for i in range(0, 20):
opts.append(2**i)
if with_cache:
gridCache = GridspecLayout(3, 10)
gridCache[0, 0] = create_Text("Cache", "warning")
gridCache[0, 1] = create_Text("Size (kB)", "warning")
gridCache[0, 2] = create_Text("Associative", "warning")
gridCache[0, 3] = create_Text("data_latency", "warning")
gridCache[1, 0] = create_Text("L1Cache", "warning")
gridCache[1, 1] = create_Dropdown("size_l1",
options=opts[:10],
value=16)
gridCache[1, 2] = create_Dropdown("assoc_l1",
options=opts[:5],
value=2)
gridCache[1, 3] = create_Dropdown("latency_l1",
options=range(14, 28),
value=16)
gridCache[2, 0] = create_Text("L2Cache", "warning")
gridCache[2, 1] = create_Dropdown("size_l2",
options=opts[6:15],
value=256)
gridCache[2, 2] = create_Dropdown("assoc_l2",
options=opts[2:10],
value=8)
gridCache[2, 3] = create_Dropdown("latency_l2",
options=range(20, 41),
value=20)
gridSim = GridspecLayout(1, 5)
gridSim[0, 0] = create_expanded_button("simulate", "Start Simulate",
"success")
display(grid)
if with_cache:
display(gridCache)
display(gridSim)
def display_html(html_string):
display(HTML(html_string))
def __init__(self,
image,
fig_xsize=None, fig_ysize=None,
cmap=plt.cm.gist_gray,
vmin=None, vmax=None,
drawtype='box'
):
deprecation_warn()
display(Markdown(f"<text style=color:blue><b>Area of Interest Selector Tips:\n</b></text>"))
display(Markdown(f'<text style=color:blue>- This plot uses "matplotlib notebook", whereas the other plots in this notebook use "matplotlib inline".</text>'))
display(Markdown(f'<text style=color:blue>- If you run this cell out of sequence and the plot is not interactive, rerun the "%matplotlib notebook" code cell.</text>'))
display(Markdown(f'<text style=color:blue>- Use the pan tool to pan with the left mouse button.</text>'))
display(Markdown(f'<text style=color:blue>- Use the pan tool to zoom with the right mouse button.</text>'))
display(Markdown(f'<text style=color:blue>- You can also zoom with a selection box using the zoom to rectangle tool.</text>'))
display(Markdown(f'<text style=color:blue>- To turn off the pan or zoom to rectangle tool so you can select an AOI, click the selected tool button again.</text>'))
display(Markdown(f'<text style=color:darkred><b>IMPORTANT!</b></text>'))
display(Markdown(f'<text style=color:darkred>- Upon loading the AOI selector, the selection tool is already active.</text>'))
display(Markdown(f'<text style=color:darkred>- Click, drag, and release the left mouse button to select an area.</text>'))
display(Markdown(f'<text style=color:darkred>- The square tool icon in the menu is <b>NOT</b> the selection tool. It is the zoom tool.</text>'))
display(Markdown(f'<text style=color:darkred>- If you select any tool, you must toggle it off before you can select an AOI</text>'))
self.image = image
self.x1 = None
self.y1 = None
self.x2 = None
self.y2 = None
if not vmin:
self.vmin = np.nanpercentile(self.image, 1)
else:
self.vmin = vmin
if not vmax:
self.vmax=np.nanpercentile(self.image, 99)
else:
self.vmax = vmax
if fig_xsize and fig_ysize:
self.fig, self.current_ax = plt.subplots(figsize=(fig_xsize, fig_ysize))
else:
self.fig, self.current_ax = plt.subplots()
self.fig.suptitle('Area-Of-Interest Selector', fontsize=16)
self.current_ax.imshow(self.image, cmap=plt.cm.gist_gray, vmin=self.vmin, vmax=self.vmax)
def toggle_selector(self, event):
print(' Key pressed.')
if event.key in ['Q', 'q'] and toggle_selector.RS.active:
print(' RectangleSelector deactivated.')
toggle_selector.RS.set_active(False)
if event.key in ['A', 'a'] and not toggle_selector.RS.active:
print(' RectangleSelector activated.')
toggle_selector.RS.set_active(True)
toggle_selector.RS = RectangleSelector(self.current_ax, self.line_select_callback,
drawtype=drawtype, useblit=True,
button=[1, 3], # don't use middle button
minspanx=0, minspany=0,
spancoords='pixels',
rectprops = dict(facecolor='red', edgecolor = 'yellow',
alpha=0.3, fill=True),
interactive=True)
plt.connect('key_press_event', toggle_selector)
def __init__(self, settings, test_name=None, default_window='hanning'):
if default_window is None:
default_window = 'None'
self.settings = settings
self.test_name = test_name
self.dataset = datastructure.DataSet()
# the 'out' construction is to refresh the text output at each update
# to stop text building up in the widget display
self.out = Output()
self.out_top = Output(layout=Layout(height='120px'))
# Initialise variables
self.current_view = 'Time'
self.N_frames = 1
self.overlap = 0.5
self.iw_fft_power = 0
self.iw_tf_power = 0
self.legend_loc = 'lower right'
# puts plot inside widget so can have buttons next to plot
self.outplot = Output(layout=Layout(width='85%'))
# BUTTONS
items_measure = [
'Log Data', 'Delete Last Measurement', 'Reset (clears all data)',
'Load Data'
]
items_view = ['View Time', 'View FFT', 'View TF']
items_calc = ['Calc FFT', 'Calc TF', 'Calc TF average']
items_axes = ['xmin', 'xmax', 'ymin', 'ymax']
items_save = ['Save Dataset', 'Save Figure']
items_iw = ['multiply iw', 'divide iw']
items_legend = ['Left', 'on/off', 'Right']
self.buttons_measure = [
Button(description=i, layout=Layout(width='33%'))
for i in items_measure
]
self.buttons_measure[0].button_style = 'success'
self.buttons_measure[0].style.font_weight = 'bold'
self.buttons_measure[1].button_style = 'warning'
self.buttons_measure[1].style.font_weight = 'bold'
self.buttons_measure[2].button_style = 'danger'
self.buttons_measure[2].style.font_weight = 'bold'
self.buttons_measure[3].button_style = 'primary'
self.buttons_measure[3].style.font_weight = 'bold'
self.buttons_view = [
Button(description=i, layout=Layout(width='95%'))
for i in items_view
]
self.buttons_view[0].button_style = 'info'
self.buttons_view[1].button_style = 'info'
self.buttons_view[2].button_style = 'info'
self.buttons_calc = [
Button(description=i, layout=Layout(width='99%'))
for i in items_calc
]
self.buttons_calc[0].button_style = 'primary'
self.buttons_calc[1].button_style = 'primary'
self.buttons_calc[2].button_style = 'primary'
self.buttons_iw_fft = [
Button(description=i, layout=Layout(width='50%')) for i in items_iw
]
self.buttons_iw_fft[0].button_style = 'info'
self.buttons_iw_fft[1].button_style = 'info'
self.buttons_iw_tf = [
Button(description=i, layout=Layout(width='50%')) for i in items_iw
]
self.buttons_iw_tf[0].button_style = 'info'
self.buttons_iw_tf[1].button_style = 'info'
self.buttons_match = Button(description='Match Amplitudes',
layout=Layout(width='99%'))
self.buttons_match.button_style = 'info'
self.buttons_save = [
Button(description=i, layout=Layout(width='50%'))
for i in items_save
]
self.buttons_save[0].button_style = 'success'
self.buttons_save[0].style.font_weight = 'bold'
self.buttons_save[1].button_style = 'success'
self.buttons_save[1].style.font_weight = 'bold'
self.button_warning = Button(
description=
'WARNING: Data may be clipped. Press here to delete last measurement.',
layout=Layout(width='100%'))
self.button_warning.button_style = 'danger'
self.button_warning.style.font_weight = 'bold'
self.button_X = Button(description='Auto X',
layout=Layout(width='95%'))
self.button_Y = Button(description='Auto Y',
layout=Layout(width='95%'))
self.button_X.button_style = 'success'
self.button_Y.button_style = 'success'
self.buttons_legend = [
Button(description=i, layout=Layout(width='31%'))
for i in items_legend
]
self.buttons_legend_toggle = ToggleButton(description='Click-and-drag',
layout=Layout(
width='95%',
alignitems='start'))
# TEXT/LABELS/DROPDOWNS
self.item_iw_fft_label = Label(value='iw power={}'.format(0),
layout=Layout(width='100%'))
self.item_iw_tf_label = Label(value='iw power={}'.format(0),
layout=Layout(width='100%'))
self.item_label = Label(value="Frame length = {:.2f} seconds.".format(
settings.stored_time /
(self.N_frames - self.overlap * self.N_frames + self.overlap)))
self.item_axis_label = Label(value="Axes control:",
layout=Layout(width='95%'))
self.item_view_label = Label(value="View data:",
layout=Layout(width='95%'))
self.item_legend_label = Label(value="Legend position:",
layout=Layout(width='95%'))
self.item_blank_label = Label(value="", layout=Layout(width='95%'))
self.text_axes = [
FloatText(value=0, description=i, layout=Layout(width='95%'))
for i in items_axes
]
self.text_axes = [self.button_X] + [self.button_Y] + self.text_axes
self.drop_window = Dropdown(options=['None', 'hanning'],
value=default_window,
description='Window:',
layout=Layout(width='99%'))
self.slide_Nframes = IntSlider(value=1,
min=1,
max=30,
step=1,
description='N_frames:',
continuous_update=True,
readout=False,
layout=Layout(width='99%'))
self.text_Nframes = IntText(value=1,
description='N_frames:',
layout=Layout(width='99%'))
# VERTICAL GROUPS
group0 = VBox([
self.buttons_calc[0], self.drop_window,
HBox(self.buttons_iw_fft)
],
layout=Layout(width='33%'))
group1 = VBox([
self.buttons_calc[1], self.drop_window, self.slide_Nframes,
self.text_Nframes, self.item_label,
HBox(self.buttons_iw_tf), self.buttons_match
],
layout=Layout(width='33%'))
group2 = VBox([
self.buttons_calc[2], self.drop_window,
HBox(self.buttons_iw_tf), self.buttons_match
],
layout=Layout(width='33%'))
group_view = VBox(
[self.item_axis_label] + self.text_axes +
[self.item_legend_label, self.buttons_legend_toggle] +
[HBox(self.buttons_legend), self.item_view_label] +
self.buttons_view,
layout=Layout(width='20%'))
# ASSEMBLE
display(self.out_top)
display(HBox([self.button_warning]))
display(HBox(self.buttons_measure))
display(HBox([self.outplot, group_view]))
display(HBox([group0, group1, group2]))
display(HBox(self.buttons_save))
self.button_warning.layout.visibility = 'hidden'
# second part to putting plot inside widget
with self.outplot:
self.p = plotting.PlotData(figsize=(7.5, 4))
## Make buttons/boxes interactive
self.text_axes[2].observe(self.xmin, names='value')
self.text_axes[3].observe(self.xmax, names='value')
self.text_axes[4].observe(self.ymin, names='value')
self.text_axes[5].observe(self.ymax, names='value')
self.button_X.on_click(self.auto_x)
self.button_Y.on_click(self.auto_y)
self.buttons_legend[0].on_click(self.legend_left)
self.buttons_legend[1].on_click(self.legend_onoff)
self.buttons_legend[2].on_click(self.legend_right)
self.buttons_legend_toggle.observe(self.legend_toggle)
self.slide_Nframes.observe(self.nframes_slide)
self.text_Nframes.observe(self.nframes_text)
self.buttons_measure[0].on_click(self.measure)
self.buttons_measure[1].on_click(self.undo)
self.buttons_measure[2].on_click(self.reset)
self.buttons_measure[3].on_click(self.load_data)
self.buttons_view[0].on_click(self.view_time)
self.buttons_view[1].on_click(self.view_fft)
self.buttons_view[2].on_click(self.view_tf)
self.buttons_calc[0].on_click(self.fft)
self.buttons_calc[1].on_click(self.tf)
self.buttons_calc[2].on_click(self.tf_av)
self.buttons_iw_fft[0].on_click(self.xiw_fft)
self.buttons_iw_fft[1].on_click(self.diw_fft)
self.buttons_iw_tf[0].on_click(self.xiw_tf)
self.buttons_iw_tf[1].on_click(self.diw_tf)
self.buttons_match.on_click(self.match)
self.buttons_save[0].on_click(self.save_data)
self.buttons_save[1].on_click(self.save_fig)
self.button_warning.on_click(self.undo)
self.refresh_buttons()
# Put output text at bottom of display
display(self.out)
with self.out_top:
try:
streams.start_stream(settings)
self.rec = streams.REC
except:
print('Data stream not initialised.')
print(
'Possible reasons: pyaudio or PyDAQmx not installed, or acquisition hardware not connected.'
)
print('Please note that it won' 't be possible to log data.')
def deploy(self, engine, streams=None):
"""
Deploy this model to an engine using the specified streams.
If `streams` is None, then streams using the REST transport
will be automatically created.
:param engine: The Engine instance to use.
:param streams: A dictionary of fastscore.Stream objects,
keyed by the slot to use, e.g.,
{ 0: input_schema, 1: output_schema}
"""
progress = Model.Ghost()
progress.value = 0
max_progress = 0
if streams is None:
max_progress += 3 * len(
self.options.schema) # create stream, verify, attach
else:
max_progress += 2 * len(streams) # verify, attach
max_progress += 2 # verify and load_model
try:
get_ipython().__class__.__name__
from ipywidgets import IntProgress
from IPython.display import display
progress = IntProgress(min=0, max=max_progress)
display(progress)
except Exception: # we're not in IPython
pass
engine.reset() # reset the engine
self.update()
try:
engine.load_model(self, dry_run=True)
progress.value += 1
except FastScoreError as e:
raise FastScoreError("Model load failed.\n{}".format(e))
streams_dict = {}
if streams is None: # Then we build the stream_dict ourselves.
for slot in self.options.schema:
sch_name = self.options.schema[slot]
schema = None
if sch_name in self.schemas:
schema = self.schemas[sch_name]
schema.update(model_manage=self._mm)
elif sch_name in self._mm.schemas.names():
schema = self._mm.schemas[sch_name]
else:
raise FastScoreError(
"Unable to find schema: {}".format(sch_name))
try:
index = engine.verify_schema(schema)
engine.unverify_schema(index)
except FastScoreError as e:
raise FastScoreError("Error in schema {}: {}".format(
schema.name, e))
stream_desc = {
"Transport": {
"Type": "REST",
"Mode": "chunked" # TODO: change to simple
},
"Encoding": "json",
"Envelope": "delimited", # TODO: change to None
"Schema": {
"$ref": sch_name
}
}
if slot in self.options.recordsets:
if self.options.recordsets[slot] == 'yes':
stream_desc["Batching"] = "explicit"
if '$all' in self.options.recordsets:
if self.options.recordsets['$all'] == 'yes':
stream_desc["Batching"] = "explicit"
if '$in' in self.options.recordsets and slot % 2 == 0:
if self.options.recordsets['$in'] == 'yes':
stream_desc["Batching"] = "explicit"
if '$out' in self.options.recordsets and slot % 2 != 0:
if self.options.recordsets['$out'] == 'yes':
stream_desc["Batching"] = "explicit"
stream_name = None
if six.PY2:
stream_name = hashlib.md5(
json.dumps(stream_desc)).hexdigest()
else:
stream_name = hashlib.md5(
json.dumps(stream_desc).encode('utf-8')).hexdigest()
stream = Stream(stream_name,
stream_desc,
model_manage=self._mm)
streams_dict[slot] = stream
progress.value += 1
else: # streams was not None.
streams_dict = streams
for slot in streams_dict: # Verify and attach each stream.
stream = streams_dict[slot]
stream.update(model_manage=self._mm)
progress.value += 1
if slot in engine.active_streams:
engine.active_streams[slot].detach()
try:
stream.attach(engine, slot, dry_run=True)
except FastScoreError as e:
raise FastScoreError(
"Unable to attach stream to slot {}.\nReason: {}".format(
slot, e.caused_by.body))
stream.attach(engine, slot)
progress.value += 1
engine.load_model(self)
progress.value += 1
return
stIdx = np.random.randint(
0, len(validation_dataset) - args.batch_size)
v_text, v_extra_info, v_len, v_label = validation_dataset[stIdx: stIdx +
args.batch_size]
v_text, v_extra_info, v_len, v_label = v_text.to(device), v_extra_info.to(device), v_len.to(device), v_label.to(device)
start_t = time.time()
# trainer.test_step(((v_text, v_extra_info),v_len), v_label)
trainer.test_step(((v_text, v_extra_info),None), v_label)
end_t = time.time()
print('| Epoch [%d] | Validation | Step [%d] | Loss: [%.4f] | Acc: [%.4f] | Time: %.1fs' %
(epoch, allStep, trainer.loss.item(), trainer.accuracy.item(), end_t - start_t))
trainer.calculateAverage()
clear_output()
print("TrainConfusion Matrix: \n")
display(pd.DataFrame(trainer.cms['Train'][-1]))
print("ValConfusion Matrix: \n")
display(pd.DataFrame(trainer.cms['Val'][-1]))
trainer.plot_all(args.model_name)
# After every Epoch, if can be moved
epoch += 1
trainer.model_save(epoch)
if args.earlyStopEpoch:
if epoch >= args.earlyStopEpoch:
break
def showSeq(normalize, step, caption, data, relDataArray=[], revert=False, oneD=False, dpi=1, save="", sideView=False,vmin=None, vmax=None,normType=matplotlib.colors.NoNorm(),verbose=True):
'''
Data should be: [batch, colorchannel, seq, H, W]
'''
if type(data) is torch.Tensor:
data = data.detach().cpu().numpy()
for i in range(len(relDataArray)):
if type(relDataArray[i]) is torch.Tensor:
relDataArray[i]=relDataArray[i].detach().cpu().numpy()
if type(data) == np.ndarray:
data=np.moveaxis(data, 1, -1)
for i in range(len(relDataArray)):
if type(relDataArray[i]) != np.ndarray:
print("Not consistent data type")
return
else:
relDataArray[i]=np.moveaxis(relDataArray[i], 1, -1)
else:
print("Undefined Type Error")
return
if verbose:
print("Data: min and max",data.min(),data.max())
for i in range(len(relDataArray)):
print("Data[",i,"]: min and max",relDataArray[i].min(),relDataArray[i].max())
if data.shape[3] == 1 and not oneD:
dimsqrt = int(math.sqrt(data.shape[2]))
if(dimsqrt*dimsqrt == data.shape[2]):
data = data.reshape(
(data.shape[0], data.shape[1], dimsqrt, dimsqrt, data.shape[4]))
else:
print("Error while reshaping")
return
#Normilize
if(vmax==None and vmin==None):
if normalize:
maxAbsVal=max(abs(data.max()),abs(data.min()))+0.00000001
if(len(relDataArray)>0):
for i in range(len(relDataArray)):
maxAbsVal=max(maxAbsVal,max(abs(relDataArray[i].max()),abs(relDataArray[i].min())))
data=((data/maxAbsVal)/2)+0.5
if(len(relDataArray)>0):
for i in range(len(relDataArray)):
relDataArray[i]=((relDataArray[i]/maxAbsVal)/2)+0.5
else:
maxAbsVal=max(abs(data.max()),abs(data.min()))+0.00000001
if(len(relDataArray)>0):
for i in range(len(relDataArray)):
maxAbsVal=max(maxAbsVal,max(abs(relDataArray[i].max()),abs(relDataArray[i].min())))
data=((data/maxAbsVal))
if(len(relDataArray)>0):
for i in range(len(relDataArray)):
relDataArray[i]=((relDataArray[i]/maxAbsVal))
# Check validity of sideview
sideView = sideView and oneD and len(relDataArray) > 0
for i in range(len(relDataArray)):
sideView = sideView and (data.shape == relDataArray[i].shape)
mData = data
if sideView:
caption = caption+"(SideView)"
div = 2+len(relDataArray)
imgConcat = torch.ones(data.shape[0], (2+len(relDataArray))
* data.shape[1], data.shape[2], data.shape[3], data.shape[4])
for cic in range(imgConcat.shape[1]):
if cic % div == 0:
imgConcat[:, cic, :, :, :] = data[:, cic//div, :, :, :]
elif cic % div < div-1:
imgConcat[:, cic, :, :, :] = relDataArray[(
cic % div)-1][:, cic//div, :, :, :]
mData = imgConcat
else:
if(len(relDataArray) > 0):
space = np.ones((data[0].shape[0], max(
1, data[0].shape[1]//20), data[0].shape[2], data[0].shape[3]))
for i in range(len(relDataArray)):
if relDataArray[i].shape[3] == 1 and not oneD:
dimsqrt = int(math.sqrt(relDataArray[i].shape[2]))
if(dimsqrt*dimsqrt == relDataArray[i].shape[2]):
relDataArray[i] = relDataArray[i].reshape(
(relDataArray[i].shape[0], relDataArray[i].shape[1], dimsqrt, dimsqrt, relDataArray[i].shape[4]))
else:
print("Error while reshaping")
return
if relDataArray[i].shape == data.shape:
mData = np.array(
[np.hstack([mData[j], space, relDataArray[i][j]]) for j in range(len(data))])
data = mData
display (Markdown('<b><span style="color: #ff0000">'+caption + (" (N)" if normalize else "") +
(" (r)" if revert else "")+"</span></b> "+(str(data.shape) if verbose else "")+''))
cmap = 'gray'
imgs_combArrey = []
for b in range(data.shape[0]):
if oneD:
imgs_combArrey.append(np.vstack([data[b, i, :, :, 0].reshape(
data.shape[2]*data.shape[3]) for i in range(data.shape[1])]))
else:
imgs_combArrey.append(np.hstack([data[b, i//2, :, :, 0] if i % 2 == 0 else np.ones(
(data.shape[2], 1)) for i in range(data.shape[1]*2-1)]))
if(b < data.shape[0]-1):
imgs_combArrey.append(np.ones((1, imgs_combArrey[-1].shape[1])))
finalImg = np.vstack(imgs_combArrey)
if revert:
if(vmax==None and vmin==None):
if normalize:
mid = 0.5
else:
mid = (finalImg.max()-finalImg.min())/2
finalImg = ((finalImg-mid)*-1)+mid
else:
mid = (vmax-vmin)/2
finalImg = ((finalImg-mid)*-1)+mid
dpi = 240.*dpi
if verbose:
print("Image: min and max",finalImg.min(),finalImg.max())
tImg = torch.from_numpy(finalImg)
plt.figure(figsize=None, dpi=dpi)
plt.imshow(finalImg, cmap=cmap, norm=normType,vmin=vmin, vmax=vmax,aspect='equal')
plt.axis('off')
# plt.colorbar()
plt.show()
if save!="":
plt.imsave(save+"/"+caption+str(step)+'.png',finalImg, cmap=cmap)
return {"name":caption,"image_data":tImg}
def visualize_tweet(tweet, prior_probs, token_probs):
"""
Visualizes a tweet and its probabilities in an IPython notebook.
Input:
tweet: a tweet as a string
prior_probs: priors for each category
token_probs: a dictionary of Counters that contain the unigram
probabilities for each category
"""
# boileplate HTML part 1
html = """
<div id="viz-overlay" style="display:none;position:absolute;width:250px;height:110px;border: 1px solid #000; padding:8px; background: #eee;">
<p>
<span style="color:orange;">P(<span class="viz-token-placeholder"></span> | food) = <span id="viz-p-food"></span></span><br>
<span style="color:blue;">P(<span class="viz-token-placeholder"></span> | water) = <span id="viz-p-water"></span><br>
<span style="color:green;">P(<span class="viz-token-placeholder"></span> | medical) = <span id="viz-p-medical"></span><br>
<span style="color:red;">P(<span class="viz-token-placeholder"></span> | energy) = <span id="viz-p-energy"></span><br>
<span style="color:gray;">P(<span class="viz-token-placeholder"></span> | none) = <span id="viz-p-none"></span></p>
</p>
</div>
<div id="viz-tweet" style="padding: 190px 0 0;">
"""
tokens = tweet.tokenList
categories = ["None", "Food", "Medical", "Energy", "Water"]
for token in tokens:
probs = [
token_probs['None'][token], token_probs['Food'][token],
token_probs['Medical'][token], token_probs['Energy'][token],
token_probs['Water'][token]
]
idx = np.argmax(probs) if sum(probs) > 0 else 0
max_class = categories[idx]
html += '<span style="%s" class="viz-token" data-food="%f" data-none="%f" data-medical="%f" data-energy="%f" data-water="%f">%s</span> ' \
% (class2color_style(max_class), token_probs['Food'][token], token_probs['None'][token], token_probs['Medical'][token],
token_probs['Energy'][token], token_probs['Water'][token], token)
# Predicted category.
predicted_category = classify_nb(tweet, prior_probs, token_probs)
html += '<p><strong>Predicted category: </strong> <span style="%s"> %s</span><br>' \
% (class2color_style(predicted_category), predicted_category)
html += '<strong>True category: </strong> <span style="%s"> %s</span></p>' \
% (class2color_style(tweet.category), tweet.category)
#Javascript
html += """
</div>
<script type="text/javascript">
$(document).ready(function() {
$("span.viz-token").mouseover(function() {
$("span.viz-token").css({"font-weight": "normal"});
$(this).css({"font-weight": "bold"});
$("span.viz-token-placeholder").text($(this).text());
$("#viz-p-food").text($(this).data("food"));
$("#viz-p-water").text($(this).data("water"));
$("#viz-p-medical").text($(this).data("medical"));
$("#viz-p-energy").text($(this).data("energy"));
$("#viz-p-none").text($(this).data("none"));
$("#viz-overlay").show();
$("#viz-overlay").offset({left:$(this).offset().left-110+$(this).width()/2, top:$(this).offset().top - 140});
});
});
</script>
"""
display(HTML(html))
def Logica(line, cell, run_query):
"""Running Logica predicates and storing results."""
predicates = ParseList(line)
if not predicates:
ShowError('No predicates to run.')
return
try:
program = ';\n'.join(s for s in [PREAMBLE, cell] if s)
parsed_rules = parse.ParseFile(program)['rule']
except parse.ParsingException as e:
e.ShowMessage()
return
try:
program = universe.LogicaProgram(parsed_rules)
except functors.FunctorError as e:
e.ShowMessage()
return
engine = program.annotations.Engine()
if engine == 'bigquery' and not BQ_READY:
ShowError(
'BigQuery client and/or authentification is not installed. \n'
'It is the easiest to run BigQuery requests from Google CoLab:\n'
' https://colab.research.google.com/.\n'
'Note that running Logica on SQLite requires no installation.\n'
'This could be a good fit for working with small data or learning Logica.\n'
'Use {warning}@Engine("sqlite");{end} annotation in your program to use SQLite.'
)
return
bar = TabBar(predicates + ['(Log)'])
logs_idx = len(predicates)
executions = []
sub_bars = []
ip = IPython.get_ipython()
for idx, predicate in enumerate(predicates):
with bar.output_to(logs_idx):
try:
sql = program.FormattedPredicateSql(predicate)
executions.append(program.execution)
ip.push({predicate + '_sql': sql})
except rule_translate.RuleCompileException as e:
print('Encountered error when compiling %s.' % predicate)
e.ShowMessage()
return
# Publish output to Colab cell.
with bar.output_to(idx):
sub_bar = TabBar(['SQL', 'Result'])
sub_bars.append(sub_bar)
with sub_bar.output_to(0):
if SHOW_FULL_QUERY:
print(
color.Format(
'The following query is stored at {warning}%s{end} '
'variable.' % (predicate + '_sql')))
print(sql)
else:
print('Query is stored at %s variable.' %
color.Warn(predicate + '_sql'))
with bar.output_to(logs_idx):
if engine == 'sqlite':
sql_runner = SqliteRunner()
elif engine == 'psql':
sql_runner = PostgresRunner()
elif engine == 'bigquery':
EnsureAuthenticatedUser()
sql_runner = RunSQL
else:
raise Exception(
'Logica only supports BigQuery, PostgreSQL and SQLite '
'for now.')
result_map = concertina_lib.ExecuteLogicaProgram(executions,
sql_runner=sql_runner,
sql_engine=engine)
for idx, predicate in enumerate(predicates):
t = result_map[predicate]
ip.push({predicate: t})
with bar.output_to(idx):
with sub_bars[idx].output_to(1):
if run_query:
print(
color.Format(
'The following table is stored at {warning}%s{end} '
'variable.' % predicate))
display(t)
else:
print('The query was not run.')
print(' ') # To activate the tabbar.
clear_output(wait=True)
display_fn(examples[current_index])
def add_annotation(annotation):
annotations.append((examples[current_index], annotation))
show_next()
def previous(btn):
show_previous()
def skip(btn):
show_next()
count_label = HTML()
set_label_text()
display(count_label)
if type(options) == list:
task_type = 'classification'
elif type(options) == tuple and len(options) in [2, 3]:
task_type = 'regression'
elif options is None:
task_type = 'captioning'
else:
raise Exception('Invalid options')
buttons = []
if task_type == 'classification':
use_dropdown = len(options) > 15
def synth(f, duration):
t = np.linspace(0., duration, int(rate * duration))
x = np.sin(f * 2. * np.pi * t)
display(Audio(x, rate=rate, autoplay=True))
# Change the following parameters as needed
write_sheet = True # #Make True to enable writes of csv files to the directory tree
#Working states
states = [
"AK", "AL", "AR", "CA", "CT", "DC", "FL", "GA", "GU", "HI", "IL", "IN",
"KY", "LA", "MA", "MN", "MS", "MT", "NC", "NE", "NH", "NM", "NY", "OR",
"PA", "TN", "TX", "VA", "VT", "WA"
]
#failing states - standalone run
#states = ["DE","ID","MD","ME","MO","RI","UT","WI","WY"]
#states using drivers
#states = ["AK","MO","HI","ID","NC","NH","WI","WY"]
#states=["AL"]
# You should not need to edit below this line
maindir = 'crdt_' + date.today().strftime('%m%d%y')
if os.path.exists(maindir):
if os.path.exists(maindir + '_old'):
shutil.rmtree(maindir + '_old')
shutil.move(maindir, maindir + '_old')
for state in states:
print("\n")
display("***" + state + " Output:***")
func = globals()["run" + state]
func(None, write_sheet)
def main():
#Recebe o dia da semana escolhido
display(Image(filename='calendario.png')
) #Colocar a imagem do calendário na mesma pasta do arquivo python
diaok = False
while not diaok:
print(
"0 - Segunda-Feira \n 1 - Terça-Feira \n 2 - Quarta-Feira \n 3 - Quinta-Feira \n 4 - Sexta-Feira \n 5 - Sábado \n 6 - Domingo"
)
dia = int(input("Digite o índice do dia da semana desejado: "))
if dia == 1 or dia == 2 or dia == 3 or dia == 4 or dia == 5 or dia == 6 or dia == 7:
diaok = True
print("================================================")
else:
print("Input inválido, escolha outro valor")
print("================================================")
#Cria lista com todos os dias do mês que forem iguais ao dia da semana escolhido
start_date = date(2018, 3, 1)
end_date = date(2018, 3, 31)
lista_dias = []
for d in daterange(start_date, end_date):
if d.weekday() == dia:
lista_dias.append(d)
#Recebe o dia do mês a ser previsto
for i, d in enumerate(lista_dias):
print('\n{} - {}'.format(i, d))
#Printa a lista dos dias disponíveis
diaok = False
while not diaok:
posicao_dia = int(
input(
"Escolha qual dos dias você prefere dentro da lista acima (apenas o índice): "
))
if posicao_dia in range(0, len(lista_dias)):
diaok = True
print("================================================")
else:
print("Input inválido, escolha outro valor de dia")
print("================================================")
#Recebe o agrupamento temporal escolhido
agrupamentook = False
while not agrupamentook:
print(
"Agrupamentos disponíveis: \n1Min \n2Min \n3Min \n4Min \n5Min \n")
agrupamento = input("Digite o agrupamento temporal desejado: ")
if agrupamento == "1Min" or agrupamento == "2Min" or agrupamento == "3Min" or agrupamento == "4Min" or agrupamento == "5Min":
agrupamentook = True
print("================================================")
else:
print("Input inválido, escolha outro valor de agrupamento")
print("================================================")
#Leitura do arquivo - Alterar com base na localização dos dados do seu computador
path = "C:\\Users\\walmart\\Documents\\USP\\TCC\\Dados\\Dados Agrupados\\Grouped by frequency"
nova_lista_dias = [
] #Nova lista reordenando os dias para que o escolhido seja o último
for i in range(0, len(lista_dias)):
if i != posicao_dia:
nova_lista_dias.append(lista_dias[i])
nova_lista_dias.append(lista_dias[posicao_dia])
#Leitura e concatenação dos arquivos dos dias
for d in nova_lista_dias:
dia = get_day(d)
if d == nova_lista_dias[0]:
dados = pd.read_csv(path + "\\" + dia + "\\" + dia + "_group_" +
str(agrupamento) + ".csv")
else:
df = pd.read_csv(path + "\\" + dia + "\\" + dia + "_group_" +
str(agrupamento) + ".csv")
dados = pd.concat([dados, df])
dados.Data = pd.to_datetime(
dados.Data) #Transforma a coluna Data em formato Data
#Recebe o radar escolhido para previsão
radares = [10426, 10433, 10482, 10484, 10492, 10500, 10521, 10531]
radarok = False
while not radarok:
print("Radares disponíveis:", radares, sep='\n')
radar = int(input("Qual o radar a ser previsto?: "))
if radar in radares:
radarok = True
print("================================================")
else:
print("Input inválido, escolha outro valor de agrupamento")
print("================================================")
dados = dados[dados['Número Agrupado'] == radar].reset_index(
drop=True) #Seleciona apenas o radar que estamos testando
#Recebe o horário divisor entre treino/teste
horaok = False
while not horaok:
hora = input(
"Formato do horário esperado: 00:00:00 \n Qual o horário de corte entre treino/teste?: "
)
if len(hora) == 8 and hora[2] == ":" and hora[5] == ":" and int(
hora[0] + hora[1]) in range(
0, 25) and int(hora[3] + hora[4]) in range(
0, 60) and int(hora[6] + hora[7]) in range(0, 60):
horaok = True
print("================================================")
else:
print("Input inválido, digite novamente o valor da hora")
print("================================================")
#CHECAR SE A SÉRIE É ESTACIONÁRIA
diferenciacao = dados.Quantidade
estacionaria = adfuller(diferenciacao)
print('ADF Statistic: %f' % estacionaria[0],
'| p-value: %f' % estacionaria[1])
if estacionaria[1] < 0.05:
print("A série do radar %s é estacionária, pois p-value < 0.05" %
radar)
pronto = True
d = 0
else:
print("A série do radar %s não é estacionária, pois p-value > 0.05" %
radar)
pronto = False
d = 0
while not pronto:
diferenciacao = diferenciacao.diff().dropna()
d = d + 1
estacionaria = adfuller(diferenciacao)
print('ADF Statistic: %f' % estacionaria[0],
'| p-value: %f' % estacionaria[1])
if estacionaria[1] < 0.05:
pronto = True
print("A ordem de diferenciação é %d" % d)
diferenciar = input("Deseja modificar o valor de d? (S/N) ")
if diferenciar == "S":
d = int(input("Digite a ordem d desejada: "))
for i in range(1, d + 1):
diferenciacao = diferenciacao.diff().dropna()
#Plotando a série original e estacionária
plt.rcParams.update({'figure.figsize': (9, 7), 'figure.dpi': 120})
fig, axis = plt.subplots(2, 1, sharex=False)
axis[0].plot(dados.Quantidade)
axis[0].set_title('Série Original')
axis[1].plot(diferenciacao)
axis[1].set_title('Série Estacionária')
plt.show()
pronto = False
while not pronto:
#ORDEM DA AUTO REGRESSÃO (PACF nos dados estacionários)
plt.rcParams.update({'figure.figsize': (9, 7), 'figure.dpi': 120})
fig, axis = plt.subplots(2, 1, sharex=False)
axis[0].set(ylim=(0, 1.05))
plot_pacf(diferenciacao, ax=axis[0], lags=100)
#ORDEM DA MÉDIA MÓVEL (ACF nos dados estacionários)
axis[1].set(ylim=(0, 1.2))
plot_acf(diferenciacao, ax=axis[1], lags=100)
plt.show()
p = int(input("Qual o valor de p adequado?: "))
q = int(input("Qual o valor de q desejado?: "))
#CONSTRUÇÃO DO MODELO
#Separação em dados de treino e de teste
dia_loc = datetime.datetime.strptime(
str(lista_dias[posicao_dia]) + " " + hora, '%Y-%m-%d %H:%M:%S'
) #Dia e hora escolhidos para separação treino/teste
divisao = dados.loc[dados['Data'] == dia_loc].index[0]
previos = int(
input(
"Quantos timelags anteriores utilizar na previsão? (Indicado acima de 1000, para entrar os dados dos dias anteriores) "
))
start = divisao - previos #Quantos steps prévios usar para a previsão
# Separa o dataframe do radar escolhido em treino e validação
treino = dados.loc[start:divisao]
teste = dados.loc[divisao:]
steps = int(
input("Quantos steps a frente das %s h deseja prever? " % hora))
teste = teste[:steps + 1]
#Indexação (horários) da previsão
data_prev = dados.loc[divisao + 1:]
data_prev = data_prev[:steps]
#Instancia o modelo e faz o fit no treino
model = ARIMA(treino['Quantidade'], order=(p, d, q))
model_fit = model.fit(disp=-1)
#Realiza o forecast
fc, se, conf = model_fit.forecast(steps, alpha=0.05) #95% de confiança
#Valores da previsão recebem o index da data_prev
fc_series = pd.Series(fc, index=data_prev.index)
lower_series = pd.Series(conf[:, 0], index=data_prev.Data)
upper_series = pd.Series(conf[:, 1], index=data_prev.Data)
start = divisao - 100 #Quantos steps prévios usar para plotar
treino = dados.loc[start:divisao]
#Plota a previsão do modelo:
fig, ax = plt.subplots(figsize=(12, 5))
ax.plot(treino['Data'],
treino['Quantidade'],
label='Observações de fato, usadas para treino')
ax.plot(teste['Data'],
teste['Quantidade'],
label='Comportamento de fato, após %s h' % hora)
ax.plot(data_prev['Data'], fc_series, label='ARIMA')
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
plt.fill_between(lower_series.index,
lower_series,
upper_series,
color='k',
alpha=0.15)
plt.title('Local: Radar %d' % radar)
plt.legend(loc='upper left', fontsize=10)
plt.show()
#TABELA DOS DADOS PREVISTOS
dados_prev = pd.Series(fc, index=data_prev.Data)
dados_prev = dados_prev.to_frame()
dados_prev.columns = ['Quantidade']
dados_prev['Data'] = dados_prev.index
dados_prev = dados_prev[['Data', 'Quantidade']]
dados_prev = dados_prev.reset_index(drop=True)
# print("Previsão:")
# print(dados_prev)
# print("")
#TABELA DOS DADOS REAIS - ALTERAR PRA PEGAR O COMPORTAMENTO REAL
dados_reais = teste[['Data', 'Quantidade']]
dados_reais = dados_reais.reset_index(drop=True)
dados_reais = dados_reais[1:]
dados_reais = dados_reais.reset_index(drop=True)
# print("Real:")
# print(dados_reais)
# print("")
#CÁLCULO DO ERRO
erros = list()
for i in range(0, 15):
erro = dados_reais.loc[i,
'Quantidade'] - dados_prev.loc[i,
'Quantidade']
erros.append(abs(erro))
erros = pd.Series(erros, index=data_prev.Data)
erros = erros.to_frame()
erros.columns = ['Erro']
erros['Data'] = erros.index
erros = erros[['Data', 'Erro']]
erros = erros.reset_index(drop=True)
# print(erros)
var_erro = statistics.variance(erros.Erro)
print("")
print("A variância do erro dessa simulação foi de: %f" % var_erro)
print("================================================")
print(model_fit.summary())
print("================================================")
#DEFINE QUAL SERIA O MODELO IDEAL PARA A PREVISÃO
modell = pm.auto_arima(dados.Quantidade,
start_p=0,
star_q=0,
test='adf',
max_p=p,
max_q=q,
m=1,
d=d,
seasonal=False,
start_P=0,
D=0,
trace=True,
error_action='ignore',
suppress_warnings=True,
stepwise=True)
print(modell.summary())
forecast_accuracy(fc, dados_reais.Quantidade)
ok = input("O modelo está adequado? (S/N) ")
if ok == "S":
pronto = True
print("Modelo definido para aplicação em outros horários")
print("================================================")
else:
print("Escolha outras ordens de modelo")
print("================================================")
#SIMULAÇÃO DE OUTROS VALORES APÓS DEFINIÇÃO DO MODELO
pronto = False
ok = input("Deseja simular? (S/N): ")
if ok == "N":
pronto = True
while not pronto:
teste = dados.loc[divisao:]
steps = int(
input("Quantos steps a frente das %s h deseja prever? " % hora))
print("================================================")
teste = teste[:steps + 1]
data_prev = dados.loc[divisao + 1:]
data_prev = data_prev[:steps]
#Realiza o forecast
fc, se, conf = model_fit.forecast(steps, alpha=0.05) #95% de confiança
#Valores da previsão recebem o index da data_prev
fc_series = pd.Series(fc, index=data_prev.index)
lower_series = pd.Series(conf[:, 0], index=data_prev.Data)
upper_series = pd.Series(conf[:, 1], index=data_prev.Data)
#Plota a previsão do modelo:
fig, ax = plt.subplots(figsize=(12, 5))
ax.plot(treino['Data'],
treino['Quantidade'],
label='Observações de fato, usadas para treino')
ax.plot(teste['Data'],
teste['Quantidade'],
label='Comportamento de fato, após %s h' % hora)
ax.plot(data_prev['Data'], fc_series, label='ARIMA')
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
plt.fill_between(lower_series.index,
lower_series,
upper_series,
color='k',
alpha=0.15)
plt.title('Local: Radar %d' % radar)
plt.legend(loc='upper left', fontsize=10)
plt.show()
ok = input("Deseja fazer outra previsão? (S/N): ")
if ok == "N":
print("Simulação encerrada")
pronto = True
else:
print("Iniciando outra simulação")
print("================================================")
def read_datalines(self, NSTEPS=0, start_step=-1, select_ckeys=None, max_vector_dim=None, even_NSTEPS=True):
"""Read NSTEPS steps of file, starting from start_step, and store only
the selected ckeys.
INPUT:
NSTEPS -> number of steps to read (default: 0 -> reads all the file)
start_step = -1 -> continue from current step (default)
0 -> go to start step
N -> go to N-th step
select_ckeys -> an array with the column keys you want to read (see all_ckeys for a list)
max_vector_dim -> when reading vectors read only this number of components (None = read all components)
even_NSTEPS -> round the number of steps to an even number (default: True)
OUTPUT:
data -> a dictionary with the selected-column steps
"""
if self._GUI:
progbar = FloatProgress(min=0, max=100)
display(progbar)
start_time = time()
if (NSTEPS == 0):
NSTEPS = self.MAX_NSTEPS
self._set_ckey(select_ckeys, max_vector_dim) # set the ckeys to read
self._initialize_dic(NSTEPS) # allocate dictionary
self.gotostep(start_step) # jump to the starting step
# read NSTEPS of the file
progbar_step = max(100000, int(0.005*NSTEPS))
for step in range(NSTEPS):
line = self.file.readline()
if (len(line) == 0): # EOF
print "Warning: reached EOF."
break
if self.endrun_keyword in line: # end-of-run keyword
print " endrun_keyword found."
break
values = np.array(line.split())
if (values.size != self.NALLCKEYS):
print "Warning: line with wrong number of columns found. Stopping here..."
print line
break
for key, idx in self.ckey.iteritems(): # save the selected columns
self.data[key][step,:] = np.array(map(float, values[idx]))
if ( (step+1)%progbar_step == 0 ):
if self._GUI:
progbar.value = float(step+1)/NSTEPS*100.;
progbar.description = "{:6.2f}%".format(progbar.value)
else:
print " step = {:9d} - {:6.2f}% completed".format(step+1, float(step+1)/NSTEPS*100.)
if self._GUI:
progbar.close()
# check number of steps read, keep an even number of steps
if (step + 1 < NSTEPS):
if (step == 0):
print "WARNING: no step read."
return
else:
if (NSTEPS != self.MAX_NSTEPS): # if NSTEPS was specified
print "Warning: less steps read."
NSTEPS = step + 1 # the correct number of read steps
# even the number of steps
if even_NSTEPS:
if (NSTEPS%2 == 1):
NSTEPS = NSTEPS - 1
for key, idx in self.ckey.iteritems(): # free the memory not used
self.data[key] = self.data[key][:NSTEPS,:]
print " ( %d ) steps read." % (NSTEPS)
self.NSTEPS = NSTEPS
print "DONE. Elapsed time: ", time()-start_time, "seconds"
return self.data
def __init__(self, *args, **kwargs):
display(Javascript(get_camera_javascript()))
super().__init__(*args, **kwargs)
def wrap(fn, *args):
"""Wrap a function with an interactive MavenWorks dashboard.
:param fn: A function to wrap, that either returns or ``display()``s some
output.
:param args: The initial values to pass to the function. Each argument
provided here will be bound to an input part.
"""
parts = {}
global_defs = []
arg_types = []
layout = {"properties": {}, "typeName": 0, "uuid": "root", "children": []}
for i, arg in enumerate(args):
arg_type = guess_type(arg)
if arg_type not in TYPE_TO_PART:
pass
arg_types.append(arg_type)
global_defs.append({
"name": "arg" + str(i),
"type": arg_type,
"value": serialize(arg, arg_type)["value"]
})
part_id = str(uuid4())
parts[part_id] = make_part(TYPE_TO_PART[arg_type], "arg" + str(i))
layout["children"].append({
"properties": {
"caption": "Arg" + str(i)
},
"typeName": 1,
"uuid": str(uuid4()),
"guid": part_id
})
new_part = make_wrapper_part(fn, arg_types)
part_id = str(uuid4())
parts[part_id] = {
"application/vnd.maven.part+json": {
"name": new_part,
"options": {
"arg" + str(i): {
"type": "Global",
"expr": "arg" + str(i),
"globals": ["arg" + str(i)]
}
for i in range(len(arg_types))
}
}
}
layout["children"].append({
"properties": {
"caption": "Output"
},
"typeName": 1,
"uuid": str(uuid4()),
"guid": part_id
})
display(
{
"application/vnd.maven.layout+json": {
"layout": layout,
"globals": global_defs,
"parts": parts
}
},
raw=True)
p22_times = p22_f(rv3.rvs(N_sims))
rv4 = uniform()
p1, p2 = np.unique(A2, return_index=True)
k23_f = interp1d(p1, Tim3[p2])
k23_times = k23_f(rv4.rvs(N_sims))
ecdf_k23 = ECDF(p22_times + k23_times) # p22_times+
dat_MH = pd.DataFrame(Tim2, columns=['Time'])
K = ecdf_k12(Tim2)
dat_MH['P1'] = K.reshape(len(Tim2), 1)
K = ecdf_k23(Tim2)
dat_MH['P2'] = K.reshape(len(Tim2), 1)
dat_MH['P1'] = transform2(dat_MH['P1'])
dat_MH['P2'] = transform2(dat_MH['P2'])
dat_MH.pop("Time")
normed_dat_MH = norm4(dat_MH)
dat_pred_MH = invtransform2(
model_MH.predict(normed_dat_MH).flatten())
rec_avg_tint = rec_avg_tint + np.concatenate(
(dat_pred1, dat_pred_MH), axis=0)
rec_avg_tint = rec_avg_tint / N_int
fin_rec[:, ii, jj] = rec_avg_tint[0]
count = count + 1
display(count / (len(IMe) * len(IMh)))
fin_rec[fin_rec > 1] = 1
## Compute the disfunctionality hazard
def show_image_in_notebook(image_path:Path):
ipythondisplay.display(ipythonimage(str(image_path)))
def _ipython_display_(self):
display(self.widget)
def display(self):
"""Display the widget."""
display(self._all_controls)
