class __Dbfs(object):
"""Database browser implementation
Args:
dbutils (DBUtils): DBUtils object (for fs only)
"""
def __init__(self, dbutils):
self.dbutils = dbutils
def create(self):
"""Create the sidecar view"""
self.sc = Sidecar(title="DBFS-%s" %
os.environ["DBJL_CLUSTER"].split("-")[-1])
self.path = "/"
self.flist = Select(options=[], rows=40, disabled=False)
self.flist.observe(self.on_click, names="value")
self.refresh = Button(description="refresh")
self.refresh.on_click(self.on_refresh)
self.output = Output()
self.up = Button(description="up")
self.up.on_click(self.on_up)
with self.sc:
display(
VBox([
HBox([self.up, self.refresh]), self.flist, self.output
]))
self.update()
def convertBytes(self, fsize):
"""Convert bytes to largest unit
Args:
fsize (int): Size in bytes
Returns:
tuple: size of largest unit, largest unit
"""
size = fsize
unit = "B"
if size > 1024 * 1024 * 1024 * 10:
size = int(size / 1024.0 / 1024.0 / 1024.0)
unit = "GB"
elif size > 1024 * 1024 * 10:
size = int(size / 1024.0 / 1024.0)
unit = "MB"
elif size > 1024 * 10:
size = int(size / 1024.0)
unit = "KB"
return (size, unit)
def update(self):
"""Update the view when an element was selected"""
with self.output:
print("updating ...")
fobjs = self.dbutils.fs.ls(self.path)
self.show_path(self.path)
dirs = sorted([fobj.name for fobj in fobjs if fobj.isDir()],
key=lambda x: x.lower())
files = sorted(
[
"%s (%d %s)" %
((fobj.name, ) + self.convertBytes(fobj.size))
for fobj in fobjs if not fobj.isDir()
],
key=lambda x: x[0].lower(),
)
self.flist.options = [""] + dirs + files
def show_path(self, path):
"""Show path in output widget
Args:
path (str): Currently selected path
"""
self.output.clear_output()
with self.output:
print("dbfs:" + re.sub(r"\s\(.*?\)$", "", path))
def on_refresh(self, b):
"""Refresh handler
Args:
b (ipywidgets.Button): clicked button
"""
self.update()
def on_up(self, b):
"""Up handler
Args:
b (ipywidgets.Button): clicked button
"""
new_path = os.path.dirname(self.path.rstrip("/"))
if new_path != self.path:
self.path = new_path
self.update()
def on_click(self, change):
"""Click handler providing db and parent as context
Args:
db (str): database name
parent (object): parent object
"""
new_path = os.path.join(self.path, change["new"])
if change["old"] is not None:
if change["new"][-1] == "/":
self.path = new_path
self.update()
else:
self.show_path(new_path)
def close(self):
"""Close view"""
self.sc.close()
class __Databases(object):
"""Database browser implementation
Args:
spark (SparkSession): Spark Session object
"""
def __init__(self, spark):
self.spark = spark
def create(self):
"""Create the sidecar view"""
self.sc = Sidecar(
title="Databases-%s" % os.environ["DBJL_CLUSTER"].split("-")[-1],
layout=Layout(width="300px"),
)
self.refresh = Button(description="refresh")
self.refresh.on_click(self.on_refresh)
self.output = Output(
layout=Layout(
height="600px", width="320px", overflow_x="scroll", overflow_y="scroll"
)
)
self.output.add_class("db-detail")
self.selects = []
self.accordion = Accordion(children=[])
with self.sc:
display(VBox([self.refresh, self.accordion, self.output]))
self.update()
self.set_css()
def on_refresh(self, b):
"""Refresh handler
Args:
b (ipywidgets.Button): clicked button
"""
self.selects = []
self.update()
def update(self):
"""Update the view when an element was selected"""
tables = {}
for obj in self.spark.sql("show tables").rdd.collect():
db = obj[0]
table = obj[1]
temp = obj[2]
if temp and db == "":
db = "temp"
if tables.get(db, None) is None:
tables[db] = []
if temp:
tables[db].append("%s (temp)" % table)
else:
tables[db].append(table)
for db in sorted(tables.keys()):
select = Select(options=[""] + sorted(tables[db]), disabled=False)
select.observe(self.on_click(db, self), names="value")
self.selects.append(select)
self.accordion.children = self.selects
for i, db in enumerate(sorted(tables.keys())):
self.accordion.set_title(i, db)
def on_click(self, db, parent):
"""Click handler providing db and parent as context
Args:
db (str): database name
parent (object): parent object
"""
def f(change):
if change["old"] is not None:
parent.output.clear_output()
with parent.output:
if db == "temp":
table = change["new"]
else:
table = "%s.%s" % (db, change["new"])
if table.endswith(" (temp)"):
table = table[:-7]
try:
schema = parent.spark.sql("describe extended %s" % table)
rows = int(parent.spark.conf.get("spark.sql.repl.eagerEval.maxNumRows"))
parent.spark.conf.set("spark.sql.repl.eagerEval.maxNumRows", 1000)
display(schema)
parent.spark.conf.set("spark.sql.repl.eagerEval.maxNumRows", rows)
except:
print("schema cannot be accessed, table most probably broken")
return f
def close(self):
"""Close view"""
self.selects = []
self.sc.close()
def set_css(self):
"""Set CSS"""
display(
HTML(
"""
<style>
.db-detail .p-Widget {
overflow: visible;
}
</style>
"""
)
)
class JS9Server(pyjs9.JS9, KeepRefs):
def __init__(self,
root='http://localhost',
path='/jjs9',
port_html=8888,
port_io=2718,
transport='socketio',
wid=None):
KeepRefs.__init__(self)
pyjs9.js9Globals['transport'] = transport
if wid is None:
self.wid = str(uuid.uuid4().hex)
else:
self.wid = wid
self.node_url = '{}:{}'.format(root, port_io)
self.frame_url = '{}:{}{}'.format(root, port_html, path)
self.displays = OrderedDict()
self.connected = False
self.msg = ''
self.id = None
#super(JS9Server, self).__init__(host=self.node_url, id=wid+'JS9')
def connect(self, wid=None, external=False):
temp = self.wid
if wid is not None:
self.wid = wid
if external:
super(JS9Server, self).__init__(host=self.node_url,
id=self.wid + 'JS9')
self.connected = True
return
if self.wid in self.displays:
super(JS9Server, self).__init__(host=self.node_url,
id=self.wid + 'JS9')
self.connected = True
else:
print('{} display does not exist'.format(self.wid))
self.wid = temp
def handler_displayed(self, widget):
#self._connect()
self.msg = 'connected'
def new_display(self,
attached=True,
wid=None,
height_px=600,
width_px=580):
if wid is not None:
self.wid = str(wid)
all_d = set()
for r in self.get_instances():
for j in r.displays.keys():
all_d.add(j)
if self.wid in all_d:
print('{} exists. Enter a different id or remove current display'.
format(self.wid))
return
html_code = "<iframe src='{}/{}' width={} height={}></iframe>".format(
self.frame_url, self.wid, width_px, height_px)
self.displays[self.wid] = {
'attached': attached,
'obj': ipw.widgets.HTML(value=html_code)
}
if attached:
display(self.displays[self.wid]['obj'])
else:
self.sc = Sidecar(title='{}'.format(self.wid),
layout=ipw.Layout(width='580px', height='600px'))
self.displays[self.wid]['obj'].on_displayed(self.handler_displayed)
with self.sc:
display(self.displays[self.wid]['obj'])
return
def close_display(self, wid=None):
if wid is not None:
closeid = wid
else:
closeid = self.wid
temp = self.displays[closeid]
if temp['attached']:
temp['obj'].close()
else:
self.sc.close()
temp['obj'].close()
#self.displays.pop(closeid)
del (self.displays[closeid])
self.connected = False
return
def close_all_displays(self, force=False):
tkeys = list(self.displays.keys())
for kid in tkeys:
self.close_display(wid=kid)
try:
self.sc.close_all()
except AttributeError:
pass
if force:
for r in self.get_instances():
for jid in r.displays.keys():
temp = r.displays[jid]
if temp['attached']:
temp['obj'].close()
else:
r.sc.close()
temp['obj'].close()
del (r.displays[jid])
r.connected = False
return