def publish_to_datasette(df, name='this', table_name='table', db_root=datasette_path):
db_path = tools.pandas_to_sqlite(df, name, table_name, db_root)
_display(_HTML(f'''<div class="alert alert-info">DataFrame sent to Jupyter Datasette folder as:
<ul>
<li><b>db</b>: {name}</li>
<li><b>table</b>: {table_name}</li>
</ul>
<i>Reload Datasette to pick up new files</i></div>'''))
def head(self, display=True, html: bool = None):
"""Return the header stats of this dataset.
If in IPython, this will be formatted to HTML. Otherwise
returns a console friendly string.
Parameters
----------
display : bool, optional
Display this header in iPython.
html : bool, optional
Generate the output as HTML.
Returns
-------
str
Header statistics.
"""
# Generate the output
if html:
fmt = ""
# HTML version
fmt += "\n"
fmt += "<table>\n"
fmt += f"<tr><th>{type(self).__name__}</th><th>Information</th></tr>\n"
row = "<tr><td>{}</td><td>{}</td></tr>\n"
# now make a call on the object to get its attributes as a list of len 2 tuples
for attr in self._get_attrs():
try:
fmt += row.format(attr[0], attr[2].format(*attr[1]))
except:
fmt += row.format(attr[0], attr[2].format(attr[1]))
if hasattr(self, 'n_arrays'):
fmt += row.format('N Arrays', self.n_arrays)
fmt += "</table>\n"
fmt += "\n"
if display:
from IPython.display import display as _display, HTML
_display(HTML(fmt))
return
return fmt
# Otherwise return a string that is Python console friendly
fmt = f"{type(self).__name__} ({hex(id(self))})\n"
# now make a call on the object to get its attributes as a list of len 2 tuples
row = " {}:\t{}\n"
for attr in self._get_attrs():
try:
fmt += row.format(attr[0], attr[2].format(*attr[1]))
except:
fmt += row.format(attr[0], attr[2].format(attr[1]))
if hasattr(self, 'n_arrays'):
fmt += row.format('N Arrays', self.n_arrays)
return fmt
def print_wolfram(wolf_query):
"""
Nicely prints a wolframAlpha query as a series of photos.
:param wolf_query: A wolfram_query() object.
:return: None.
"""
for result in wolf_query['pod']:
outer = result['subpod']
if type(outer) is dict:
disp = _Image(url=outer['img']['@src']) # type: tuple
_display(disp)
else:
for i in range(len(outer)):
disp = _Image(url=outer[i]['img']['@src']) # type: tuple
_display(disp)
def display(self):
"displays helpful doc"
tpe = self.tasktype()
des = ' '.join(getattr(_DOCHelper, self.name).value)
doc = f"""
# {str(self)}
Its task is `{tpe.__module__}.{tpe.__qualname__}`
## Description
{des.capitalize()}
## `{tpe.__qualname__}` Documentation
""".replace('\n ', '\n').strip() + '\n'
doc += tpe.__doc__.replace('\n ', '\n').replace('\n#', '\n###')
_display(Markdown(doc.strip()))
return _display()
def print_verbose_dataframe(df):
r"""
Print the full dataframe no matter how big
Parameters
----------
df
Returns
-------
"""
import pandas as _pd
from IPython.display import display as _display
with _pd.option_context('display.max_rows', None,
'display.max_columns', None,
'display.width', 1000):
_display(df)
def _iimage(self, iL: list):
"""insert one or two images from file
Args:
iL (list): image parameters
"""
utfS = ""
if "," in iL[1]: # two images
scaleF = iL[2].split(",")
scale1F = float(scaleF[0])
scale2F = float(scaleF[1])
self.setcmdD.update({"scale1F": scale1F})
self.setcmdD.update({"scale2F": scale2F})
fileS = iL[1].split(",")
file1S = fileS[0].strip()
file2S = fileS[1].strip()
docpS = "d" + self.setsectD["cnumS"]
img1S = str(Path(self.folderD["dpathcur"] / file1S))
img2S = str(Path(self.folderD["dpathcur"] / file2S))
# pshrt1S = str(Path(*Path(img1S).parts[-4:]))
# pshrt2S = str(Path(*Path(img2S).parts[-4:]))
for fS in [img1S, img2S]:
utfS += "Figure path: " + fS + "\n"
try:
_display(_Image(fS))
except:
pass
print(utfS)
self.calcS += utfS + "\n"
else: # one image
scale1F = float(iL[2])
self.setcmdD.update({"scale1F": scale1F})
fileS = iL[1].split(",")
file1S = fileS[0].strip()
docpS = "d" + self.setsectD["cnumS"]
img1S = str(Path(self.folderD["dpathcur"] / file1S))
utfS += "Figure path: " + img1S + "\n"
try:
_display(_Image(img1S))
except:
pass
print(utfS)
self.calcS += utfS + "\n"
def start_datasette(db_path, jupyter, **kwargs):
'''
Pass in cmdline flags as kwargs: port=8080, reload=True
'''
port = kwargs.get('port',kwargs.get('p',None))
host = _socket.gethostbyname(_socket.gethostname())
# keep the kwargs alive
flags = [f'--{key}' if value is True else f'--{key} {value}' for key,value in kwargs.items()]
flags = ' '.join(flags)
cmd = f'datasette {db_path} {flags}'
cmd = re.sub(' +', ' ', cmd)
if jupyter:
_display(_HTML(f'''<div class="alert alert-info><b>Datasette Launching at <a target="_blank" href="http://{host}:{port}">http://{host}:{port}</a></b><br/>
<pre>{cmd}</pre></div>'''))
try:
process = _subprocess.Popen(cmd.split(' '), stderr=_subprocess.PIPE, stdout=_subprocess.PIPE)
return process
except: #catch KeyboardInterrupt and whatever other calamity might befall our process
process.terminate()
def print_color_bold(string, color='black'):
text_line = _Markdown("<span style=\"color: " + color + "\">**" + string + "**</span>") # type: tuple
_display(text_line)
def _create_animation(od, time, plot_func, func_kwargs, display, **kwargs):
"""
Create animation using oceanspy plot functions.
Parameters
----------
od: OceanDataset
oceandataset used to plot.
time: DataArray
DataArray corresponding to time.
plot_func: function
Alias referring to the plot function.
func_kwargs:
Keyword arguments for plot_func.
display: bool
If True, display the animation.
**kwargs:
Keyword arguments for py:class:`matplotlib.animation.FuncAnimation`
Returns
-------
anim: matplotlib.animation.FuncAnimation
Animation object
References
----------
https://matplotlib.org/api/_as_gen/matplotlib.animation.FuncAnimation.html
"""
# Check parameters
_check_instance(
{
"od": od,
"time": time,
"func_kwargs": func_kwargs,
"display": display
},
{
"od": "oceanspy.OceanDataset",
"time": "xarray.DataArray",
"func_kwargs": ["type(None)", "dict"],
"display": ["bool"],
},
)
# Handle kwargs
if func_kwargs is None:
func_kwargs = {}
func_kwargs = dict(func_kwargs)
# Animate function
def animate(i):
_plt.clf()
func_kwargs["cutout_kwargs"] = {
"timeRange": time.isel({
time.dims[0]: i
}).values,
"dropAxes": "time",
}
with _io.capture_output() as captured:
plot_func(od, **func_kwargs)
if "pbar" in locals():
pbar.update(1)
# Create animation object
anim = _FuncAnimation(**{
"fig": _plt.gcf(),
"func": animate,
"frames": len(time),
**kwargs
})
# Display
if display is True:
pbar = _tqdm(total=len(time))
_display(_HTML(anim.to_html5_video()))
pbar.close()
del pbar
return anim
def _get_ticks(self, asset='EURUSD', start=None, end=None,
cond=None, verbose=False, side='both',
separated=False, fill=True, darwinex_time=False):
if asset not in self.available_assets:
raise KeyError('Asset {} not available'.format(asset))
if cond is None:
_files_df = self.list_of_files(asset)[start: end]
else:
_files_df = self.list_of_files(asset)[cond]
if verbose is True:
print("\n[INFO] Retrieving data from Darwinex Tick Data "
"Server..")
side = side.upper()
posits = ['ASK', 'BID']
max_bar = _files_df.shape[0]
if side in posits:
posits = [side]
max_bar /= 2
data = {}
if _isnotebook():
if 'ipywidgets' in _modules:
from ipywidgets import FloatProgress as _FloatProgress
from IPython.display import display as _display
progressbar = _FloatProgress(min=0, max=max_bar)
if max_bar > 1:
_display(progressbar)
elif self._widgets_available:
print(
'You must install ipywidgets module to display progress bar '
'for notebooks. Use "pip install ipywidgets"')
self._widgets_available = False
else:
self._widgets_available = False
right_download = 0
wrong_download = 0
for posit in posits:
_files = _files_df[_files_df.pos == posit]['file'].values
_files = ['{}/{}'.format(asset, f) for f in _files]
data_rec = []
# print(_files)
for _file in _files:
pos = 'Ask' if 'ASK' in _file else 'Bid'
try:
data_pro = None
for retry in range(self.num_retries):
self._get_file(_file)
# Construct DataFrame
data_pro = [
_pd.read_table(self._virtual_dl, compression='gzip',
sep=',', header=None,
lineterminator='\n',
names=['Time', pos, pos + '_size'],
index_col='Time', parse_dates=[0],
date_parser=self._parser)]
if len(data_pro) > 0:
right_download += 1
break
else:
sleep(self.await_time)
if verbose:
print('File missing, retrying.')
data_rec += data_pro
if verbose is True:
print('Downloaded file {}'.format(_file))
# Case: if file not found
except Exception as ex:
_exstr = "\nException Type {0}. Args:\n{1!r}"
_msg = _exstr.format(type(ex).__name__, ex.args)
print(_msg)
wrong_download += 1
if self._widgets_available:
progressbar.value += 1
else:
try:
print('*', end="", flush=True),
except TypeError:
print('*', end=""),
data[posit] = _pd.concat(data_rec, sort=True, axis=0,
verify_integrity=False)
if len(posits) == 2:
if not separated:
data = _pd.concat([data[posit] for posit in posits], axis=1)
if fill:
data = data.ffill()
else:
data = data[posits[0]]
print('Process completed. {} files downloaded'.format(right_download))
if wrong_download > 0:
print('{} files could not be downloaded.'.format(wrong_download))
if darwinex_time:
data.index = _index_utc_to_mt_time(data.index)
return data
def display_weights(w):
_display(w.to_frame().transpose())
def display_re_matrix(re):
_display(re.unstack(level=0).sort_values(by=0))
def _create_animation(od, time, plot_func, func_kwargs, display,
**kwargs):
"""
Create animation using oceanspy plot functions.
Parameters
----------
od: OceanDataset
oceandataset used to plot.
time: DataArray
DataArray corresponding to time.
plot_func: function
Alias referring to the plot function.
func_kwargs:
Keyword arguments for plot_func.
display: bool
If True, display the animation.
**kwargs:
Keyword arguments for py:class:`matplotlib.animation.FuncAnimation`
Returns
-------
anim: matplotlib.animation.FuncAnimation
Animation object
References
----------
https://matplotlib.org/api/_as_gen/matplotlib.animation.FuncAnimation.html
"""
# Check parameters
_check_instance({'od': od,
'time': time,
'func_kwargs': func_kwargs,
'display': display},
{'od': 'oceanspy.OceanDataset',
'time': 'xarray.DataArray',
'func_kwargs': ['type(None)', 'dict'],
'display': ['bool']})
# Handle kwargs
if func_kwargs is None:
func_kwargs = {}
func_kwargs = dict(func_kwargs)
# Animate function
def animate(i):
_plt.clf()
func_kwargs['cutout_kwargs'] = {'timeRange':
time.isel({time.dims[0]: i}).values,
'dropAxes': 'time'}
with _io.capture_output() as captured:
plot_func(od, **func_kwargs)
if 'pbar' in locals():
pbar.update(1)
# Create animation object
anim = _FuncAnimation(**{'fig': _plt.gcf(),
'func': animate,
'frames': len(time),
**kwargs})
# Display
if display is True:
pbar = _tqdm(total=len(time))
_display(_HTML(anim.to_html5_video()))
pbar.close()
del pbar
return anim
def _create_animation(od, time, plot_func, func_kwargs, display, **kwargs):
"""
Create animation using oceanspy plot functions.
Parameters
----------
od: OceanDataset
oceandataset to check for missing variables
time: DataArray
DataArray corresponding to time
plot_func: function
Alias referring to the plot function
func_kwargs:
Keyword arguments for plot_func
display: bool
If True, display the animation
**kwargs:
Keyword arguments for matplotlib.animation.FuncAnimation
Returns
-------
Animation object
See also
--------
subsample.coutout
References
----------
https://matplotlib.org/api/_as_gen/matplotlib.animation.FuncAnimation.html
"""
# Check input
if not isinstance(od, _ospy.OceanDataset):
raise TypeError('`od` must be OceanDataset')
if not isinstance(time, _xr.DataArray):
raise TypeError('`time` must be a DataArray')
elif len(time.dims) != 1:
raise TypeError('`time` must have one dimension only')
# TODO: check plot function
if not isinstance(func_kwargs, (type(None), dict)):
raise TypeError('`func_kwargs` must be None or dict')
if not isinstance(display, bool):
raise TypeError('`display` must be bool')
# Handle kwargs
if func_kwargs is None: func_kwargs = {}
# Animate function
def animate(i):
_plt.clf()
func_kwargs['cutout_kwargs'] = {
'timeRange': time.isel({
time.dims[0]: i
}).values,
'dropAxes': 'time'
}
with _io.capture_output() as captured:
plot_func(od, **func_kwargs)
if 'pbar' in locals(): pbar.update(1)
# Create animation object
anim = _FuncAnimation(**{
'fig': _plt.gcf(),
'func': animate,
'frames': len(time),
**kwargs
})
# Display
if display is True:
pbar = _tqdm(total=len(time))
_display(_HTML(anim.to_html5_video()))
pbar.close()
del pbar
return anim
def printmd(string):
from IPython.display import display as _display
from IPython.display import Markdown as _Markdown
"""Displays a markdown string in the jupyter notebook"""
_display(_Markdown(string))
