def installPixiedustJar(): with pkg_resources.resource_stream(__name__, "resources/pixiedust.jar") as resJar: with open(jarFilePath, 'wb+') as installedJar: shutil.copyfileobj(resJar, installedJar) printEx("Pixiedust runtime updated. Please restart kernel", PrintColors.RED)
def display(entity, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") selectedHandler = getSelectedHandler(kwargs, entity) myLogger.debug( "Creating a new display handler with options {0}: {1}".format( kwargs, selectedHandler)) displayHandler = selectedHandler.newDisplayHandler(kwargs, entity) if displayHandler is None: printEx("Unable to obtain handler") return displayHandler.handlerMetadata = selectedHandler displayHandler.callerText = traceback.extract_stack(limit=2)[0][3] displayHandler.render()
def _doDownload(d): artifact=art[0] if not artifact.version or artifact.version=='0': artifact.version = d.resolver._find_latest_version_available(artifact) fileLoc = artifact.get_filename(self.DOWNLOAD_DIR) if os.path.isfile(fileLoc): os.remove(fileLoc) results = d.download(artifact,filename=self.DOWNLOAD_DIR) if not results[1]: raise Exception("Error downloading package {0}".format(str(artifact))) else: artifact=results[0] print("Artifact downloaded successfully {0}".format(str(artifact))) printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED) fileLoc=self.storeArtifact(artifact,base) return fileLoc
def _doDownload(d): package=packs[0] if not package.version or package.version=='0': package.version = d.resolver._find_latest_version_available(package) fileLoc = package.getFilePath(self.DOWNLOAD_DIR) if os.path.isfile(fileLoc): os.remove(fileLoc) results = d.download(package,filename=self.DOWNLOAD_DIR) if not results[1]: raise Exception("Error downloading package {0}".format(str(package))) else: package=results[0] print("Package {0} downloaded successfully".format(str(package))) printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED) fileLoc=self.storePackage(package,base) return fileLoc
def _doDownload(d): package=packs[0] if not package.version or package.version=='0': package.version = d.resolver._find_latest_version_available(package) fileLoc = package.getFilePath(self.DOWNLOAD_DIR) if os.path.isfile(fileLoc): os.remove(fileLoc) results = d.download(package,filename=self.DOWNLOAD_DIR) if not results[1]: raise Exception("Error downloading package {0}".format(str(package))) else: package=results[0] print("Package {0} downloaded successfully".format(str(package))) printEx("Please restart Kernel to complete installation of the new package",PrintColors.RED) fileLoc=self.storePackage(package,base) return fileLoc
def display(entity, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") #todo: use ConverterRegistry def toPython(entity): from py4j.java_gateway import JavaObject if entity is None or not isinstance(entity, JavaObject): return entity clazz = entity.getClass().getName() if clazz == "org.apache.spark.sql.Dataset": entity = entity.toDF() clazz = "org.apache.spark.sql.DataFrame" if clazz == "org.apache.spark.sql.DataFrame": from pyspark.sql import DataFrame, SQLContext from pyspark import SparkContext entity = DataFrame( entity, SQLContext(SparkContext.getOrCreate(), entity.sqlContext())) return entity if 'pixiedust_display_callerText' in globals(): callerText = globals()['pixiedust_display_callerText'] else: callerText = traceback.extract_stack(limit=2)[0][3] if callerText is None or callerText == "": raise Exception("Unable to get caller text") pr = None try: if "cell_id" in kwargs and "showchrome" not in kwargs and "handlerId" in kwargs: if "gen_tests" in kwargs: #remove gen_tests from command line import re m = re.search(",\\s*gen_tests\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE) if m is not None: callerText = callerText.replace(m.group(0), "") #generate new prefix p = re.search(",\\s*prefix\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE) if p is not None: prefix = ''.join( [",prefix='", str(uuid.uuid4())[:8], "'"]) callerText = callerText.replace(p.group(0), prefix) get_ipython().set_next_input(callerText) if "profile" in kwargs: import cProfile pr = cProfile.Profile() pr.enable() scalaKernel = False if callerText is None or callerText == "" and hasattr( display, "fetchEntity"): callerText, entity = display.fetchEntity(entity) entity = toPython(entity) scalaKernel = True #get a datahandler and displayhandler for this entity dataHandler = getDataHandler(kwargs, entity) selectedHandler = getSelectedHandler(kwargs, entity, dataHandler) #notify listeners of a new display Run for displayRunListener in displayRunListeners: displayRunListener(entity, kwargs) #check if we have a job monitor id from pixiedust.utils.sparkJobProgressMonitor import progressMonitor if progressMonitor: progressMonitor.onDisplayRun(kwargs.get("cell_id")) myLogger.debug( "Creating a new display handler with options {0}: {1}".format( kwargs, selectedHandler)) try: displayHandler = selectedHandler.newDisplayHandler( kwargs, entity, dataHandler) except TypeError: displayHandler = selectedHandler.newDisplayHandler( kwargs, entity) if displayHandler is None: printEx("Unable to obtain handler") return displayHandler.handlerMetadata = selectedHandler displayHandler.dataHandler = dataHandler displayHandler.callerText = callerText if scalaKernel: displayHandler.scalaKernel = True if displayHandler.callerText is None: printEx("Unable to get entity information") return displayHandler.render() finally: if pr is not None: import pstats, StringIO pr.disable() s = StringIO.StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() myLogger.debug(s.getvalue())
def display(entity, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") #todo: use ConverterRegistry def toPython(entity): from py4j.java_gateway import JavaObject if entity is None or not isinstance(entity, JavaObject): return entity clazz = entity.getClass().getName() if clazz == "org.apache.spark.sql.Dataset": entity = entity.toDF() clazz = "org.apache.spark.sql.DataFrame" if clazz == "org.apache.spark.sql.DataFrame": from pyspark.sql import DataFrame, SQLContext from pyspark import SparkContext entity = DataFrame(entity, SQLContext(SparkContext.getOrCreate(), entity.sqlContext())) return entity if 'pixiedust_display_callerText' in globals(): callerText = globals()['pixiedust_display_callerText'] else: callerText = get_caller_text(inspect.currentframe()) if callerText is None or callerText == "": raise Exception("Unable to get caller text") pr = None try: if "cell_id" in kwargs and "showchrome" not in kwargs and "handlerId" in kwargs: if "gen_tests" in kwargs: #remove gen_tests from command line import re m = re.search(",\\s*gen_tests\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE) if m is not None: callerText = callerText.replace(m.group(0),"") #generate new prefix p = re.search(",\\s*prefix\\s*=\\s*'((\\\\'|[^'])*)'", str(callerText), re.IGNORECASE) if p is not None: prefix = ''.join([",prefix='", str(uuid.uuid4())[:8], "'"]) callerText = callerText.replace(p.group(0), prefix) get_ipython().set_next_input(callerText) if "profile" in kwargs: import cProfile pr = cProfile.Profile() pr.enable() scalaKernel = False if callerText is None or callerText == "" and hasattr(display, "fetchEntity"): callerText, entity = display.fetchEntity(entity) entity = toPython(entity) scalaKernel = True #get a datahandler and displayhandler for this entity dataHandler = getDataHandler(kwargs, entity) selectedHandler = getSelectedHandler(kwargs, entity, dataHandler) #notify listeners of a new display Run for displayRunListener in displayRunListeners: displayRunListener(entity, kwargs) #check if we have a job monitor id from pixiedust.utils.sparkJobProgressMonitor import progressMonitor if progressMonitor: progressMonitor.onDisplayRun(kwargs.get("cell_id")) myLogger.debug("Creating a new display handler with options {0}: {1}".format(kwargs, selectedHandler)) try: displayHandler = selectedHandler.newDisplayHandler(kwargs,entity, dataHandler) except TypeError: displayHandler = selectedHandler.newDisplayHandler(kwargs, entity ) if displayHandler is None: printEx("Unable to obtain handler") return displayHandler.handlerMetadata = selectedHandler displayHandler.dataHandler = dataHandler displayHandler.callerText = callerText if scalaKernel: displayHandler.scalaKernel = True if displayHandler.callerText is None: printEx("Unable to get entity information") return displayHandler.render() finally: if pr is not None: import pstats, StringIO pr.disable() s = StringIO.StringIO() ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() myLogger.debug(s.getvalue())
def installPixiedustJar(): with pkg_resources.resource_stream(__name__, "resources/pixiedust.jar") as resJar: with open( jarFilePath, 'wb+' ) as installedJar: shutil.copyfileobj(resJar, installedJar) printEx("Pixiedust runtime updated. Please restart kernel",PrintColors.RED)