def installPixiedustJar():
with pkg_resources.resource_stream(__name__,
"resources/pixiedust.jar") as resJar:
with open(jarFilePath, 'wb+') as installedJar:
shutil.copyfileobj(resJar, installedJar)
printEx("Pixiedust runtime updated. Please restart kernel",
PrintColors.RED)
def display(entity, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
selectedHandler = getSelectedHandler(kwargs, entity)
myLogger.debug(
"Creating a new display handler with options {0}: {1}".format(
kwargs, selectedHandler))
displayHandler = selectedHandler.newDisplayHandler(kwargs, entity)
if displayHandler is None:
printEx("Unable to obtain handler")
return
displayHandler.handlerMetadata = selectedHandler
displayHandler.callerText = traceback.extract_stack(limit=2)[0][3]
displayHandler.render()
def _doDownload(d):
artifact=art[0]
if not artifact.version or artifact.version=='0':
artifact.version = d.resolver._find_latest_version_available(artifact)
fileLoc = artifact.get_filename(self.DOWNLOAD_DIR)
if os.path.isfile(fileLoc):
os.remove(fileLoc)
results = d.download(artifact,filename=self.DOWNLOAD_DIR)
if not results[1]:
raise Exception("Error downloading package {0}".format(str(artifact)))
else:
artifact=results[0]
print("Artifact downloaded successfully {0}".format(str(artifact)))
printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED)
fileLoc=self.storeArtifact(artifact,base)
return fileLoc
def _doDownload(d):
package=packs[0]
if not package.version or package.version=='0':
package.version = d.resolver._find_latest_version_available(package)
fileLoc = package.getFilePath(self.DOWNLOAD_DIR)
if os.path.isfile(fileLoc):
os.remove(fileLoc)
results = d.download(package,filename=self.DOWNLOAD_DIR)
if not results[1]:
raise Exception("Error downloading package {0}".format(str(package)))
else:
package=results[0]
print("Package {0} downloaded successfully".format(str(package)))
printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED)
fileLoc=self.storePackage(package,base)
return fileLoc
def _doDownload(d):
package=packs[0]
if not package.version or package.version=='0':
package.version = d.resolver._find_latest_version_available(package)
fileLoc = package.getFilePath(self.DOWNLOAD_DIR)
if os.path.isfile(fileLoc):
os.remove(fileLoc)
results = d.download(package,filename=self.DOWNLOAD_DIR)
if not results[1]:
raise Exception("Error downloading package {0}".format(str(package)))
else:
package=results[0]
print("Package {0} downloaded successfully".format(str(package)))
printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED)
fileLoc=self.storePackage(package,base)
return fileLoc
def display(entity, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
#todo: use ConverterRegistry
def toPython(entity):
from py4j.java_gateway import JavaObject
if entity is None or not isinstance(entity, JavaObject):
return entity
clazz = entity.getClass().getName()
if clazz == "org.apache.spark.sql.Dataset":
entity = entity.toDF()
clazz = "org.apache.spark.sql.DataFrame"
if clazz == "org.apache.spark.sql.DataFrame":
from pyspark.sql import DataFrame, SQLContext
from pyspark import SparkContext
entity = DataFrame(
entity,
SQLContext(SparkContext.getOrCreate(),
entity.sqlContext()))
return entity
if 'pixiedust_display_callerText' in globals():
callerText = globals()['pixiedust_display_callerText']
else:
callerText = traceback.extract_stack(limit=2)[0][3]
if callerText is None or callerText == "":
raise Exception("Unable to get caller text")
pr = None
try:
if "cell_id" in kwargs and "showchrome" not in kwargs and "handlerId" in kwargs:
if "gen_tests" in kwargs:
#remove gen_tests from command line
import re
m = re.search(",\\s*gen_tests\\s*=\\s*'((\\\\'|[^'])*)'",
str(callerText), re.IGNORECASE)
if m is not None:
callerText = callerText.replace(m.group(0), "")
#generate new prefix
p = re.search(",\\s*prefix\\s*=\\s*'((\\\\'|[^'])*)'",
str(callerText), re.IGNORECASE)
if p is not None:
prefix = ''.join(
[",prefix='",
str(uuid.uuid4())[:8], "'"])
callerText = callerText.replace(p.group(0), prefix)
get_ipython().set_next_input(callerText)
if "profile" in kwargs:
import cProfile
pr = cProfile.Profile()
pr.enable()
scalaKernel = False
if callerText is None or callerText == "" and hasattr(
display, "fetchEntity"):
callerText, entity = display.fetchEntity(entity)
entity = toPython(entity)
scalaKernel = True
#get a datahandler and displayhandler for this entity
dataHandler = getDataHandler(kwargs, entity)
selectedHandler = getSelectedHandler(kwargs, entity, dataHandler)
#notify listeners of a new display Run
for displayRunListener in displayRunListeners:
displayRunListener(entity, kwargs)
#check if we have a job monitor id
from pixiedust.utils.sparkJobProgressMonitor import progressMonitor
if progressMonitor:
progressMonitor.onDisplayRun(kwargs.get("cell_id"))
myLogger.debug(
"Creating a new display handler with options {0}: {1}".format(
kwargs, selectedHandler))
try:
displayHandler = selectedHandler.newDisplayHandler(
kwargs, entity, dataHandler)
except TypeError:
displayHandler = selectedHandler.newDisplayHandler(
kwargs, entity)
if displayHandler is None:
printEx("Unable to obtain handler")
return
displayHandler.handlerMetadata = selectedHandler
displayHandler.dataHandler = dataHandler
displayHandler.callerText = callerText
if scalaKernel:
displayHandler.scalaKernel = True
if displayHandler.callerText is None:
printEx("Unable to get entity information")
return
displayHandler.render()
finally:
if pr is not None:
import pstats, StringIO
pr.disable()
s = StringIO.StringIO()
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
ps.print_stats()
myLogger.debug(s.getvalue())
def display(entity, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
#todo: use ConverterRegistry
def toPython(entity):
from py4j.java_gateway import JavaObject
if entity is None or not isinstance(entity, JavaObject):
return entity
clazz = entity.getClass().getName()
if clazz == "org.apache.spark.sql.Dataset":
entity = entity.toDF()
clazz = "org.apache.spark.sql.DataFrame"
if clazz == "org.apache.spark.sql.DataFrame":
from pyspark.sql import DataFrame, SQLContext
from pyspark import SparkContext
entity = DataFrame(entity, SQLContext(SparkContext.getOrCreate(), entity.sqlContext()))
return entity
if 'pixiedust_display_callerText' in globals():
callerText = globals()['pixiedust_display_callerText']
else:
callerText = get_caller_text(inspect.currentframe())
if callerText is None or callerText == "":
raise Exception("Unable to get caller text")
pr = None
try:
if "cell_id" in kwargs and "showchrome" not in kwargs and "handlerId" in kwargs:
if "gen_tests" in kwargs:
#remove gen_tests from command line
import re
m = re.search(",\\s*gen_tests\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE)
if m is not None:
callerText = callerText.replace(m.group(0),"")
#generate new prefix
p = re.search(",\\s*prefix\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE)
if p is not None:
prefix = ''.join([",prefix='", str(uuid.uuid4())[:8], "'"])
callerText = callerText.replace(p.group(0), prefix)
get_ipython().set_next_input(callerText)
if "profile" in kwargs:
import cProfile
pr = cProfile.Profile()
pr.enable()
scalaKernel = False
if callerText is None or callerText == "" and hasattr(display, "fetchEntity"):
callerText, entity = display.fetchEntity(entity)
entity = toPython(entity)
scalaKernel = True
#get a datahandler and displayhandler for this entity
dataHandler = getDataHandler(kwargs, entity)
selectedHandler = getSelectedHandler(kwargs, entity, dataHandler)
#notify listeners of a new display Run
for displayRunListener in displayRunListeners:
displayRunListener(entity, kwargs)
#check if we have a job monitor id
from pixiedust.utils.sparkJobProgressMonitor import progressMonitor
if progressMonitor:
progressMonitor.onDisplayRun(kwargs.get("cell_id"))
myLogger.debug("Creating a new display handler with options {0}: {1}".format(kwargs, selectedHandler))
try:
displayHandler = selectedHandler.newDisplayHandler(kwargs,entity, dataHandler)
except TypeError:
displayHandler = selectedHandler.newDisplayHandler(kwargs, entity )
if displayHandler is None:
printEx("Unable to obtain handler")
return
displayHandler.handlerMetadata = selectedHandler
displayHandler.dataHandler = dataHandler
displayHandler.callerText = callerText
if scalaKernel:
displayHandler.scalaKernel = True
if displayHandler.callerText is None:
printEx("Unable to get entity information")
return
displayHandler.render()
finally:
if pr is not None:
import pstats, StringIO
pr.disable()
s = StringIO.StringIO()
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
ps.print_stats()
myLogger.debug(s.getvalue())
def installPixiedustJar():
with pkg_resources.resource_stream(__name__, "resources/pixiedust.jar") as resJar:
with open( jarFilePath, 'wb+' ) as installedJar:
shutil.copyfileobj(resJar, installedJar)
printEx("Pixiedust runtime updated. Please restart kernel",PrintColors.RED)