def __init__(self, shell, data=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
try:
should_serialize = conf.serialize()
if should_serialize:
self.logger.debug("Serialization enabled.")
self.magics_home_path = get_magics_home_path()
path_to_serialize = join_paths(self.magics_home_path, "state.json")
self.logger.debug("Will serialize to {}.".format(path_to_serialize))
self.spark_controller = SparkController(self.ipython_display, serialize_path=path_to_serialize)
else:
self.logger.debug("Serialization NOT enabled.")
except KeyError:
self.logger.error("Could not read env vars for serialization.")
self.logger.debug("Initialized spark magics.")
def __init__(self, shell, data=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = Log("SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
try:
should_serialize = conf.serialize()
if should_serialize:
self.logger.debug("Serialization enabled.")
self.magics_home_path = get_magics_home_path()
path_to_serialize = join_paths(self.magics_home_path,
"state.json")
self.logger.debug(
"Will serialize to {}.".format(path_to_serialize))
self.spark_controller = SparkController(
self.ipython_display, serialize_path=path_to_serialize)
else:
self.logger.debug("Serialization NOT enabled.")
except KeyError:
self.logger.error("Could not read env vars for serialization.")
self.logger.debug("Initialized spark magics.")
def __init__(self, **kwargs):
# Simply invokes the behavior of the superclass, but sets the filename keyword argument if it's not already set.
if 'filename' in kwargs:
super(MagicsFileHandler, self).__init__(**kwargs)
else:
magics_home_path = get_magics_home_path()
logs_folder_name = "logs"
log_file_name = "log_{}.log".format(get_instance_id())
directory = FileSystemReaderWriter(join_paths(magics_home_path, logs_folder_name))
directory.ensure_path_exists()
super(MagicsFileHandler, self).__init__(filename=join_paths(directory.path, log_file_name), **kwargs)
def __init__(self, **kwargs):
# Simply invokes the behavior of the superclass, but sets the filename keyword argument if it's not already set.
if 'filename' in kwargs:
super(MagicsFileHandler, self).__init__(**kwargs)
else:
magics_home_path = get_magics_home_path()
logs_folder_name = "logs"
log_file_name = "log_{}.log".format(get_instance_id())
directory = FileSystemReaderWriter(
join_paths(magics_home_path, logs_folder_name))
directory.ensure_path_exists()
super(MagicsFileHandler,
self).__init__(filename=join_paths(directory.path,
log_file_name),
**kwargs)
def load(fsrw_class=None):
"""Initializes the global configuration by reading from the configuration
file, overwriting the current set of overrides if there is one"""
if fsrw_class is None:
fsrw_class = FileSystemReaderWriter
home_path = fsrw_class(get_magics_home_path())
home_path.ensure_path_exists()
config_file = fsrw_class(join_paths(home_path.path, CONFIG_JSON))
config_file.ensure_file_exists()
config_text = config_file.read_lines()
line = "".join(config_text).strip()
if line == "":
overrides = {}
else:
overrides = json.loads(line)
override_all(overrides)
def load(fsrw_class=None):
"""Initializes the global configuration by reading from the configuration
file, overwriting the current set of overrides if there is one"""
if fsrw_class is None:
fsrw_class = FileSystemReaderWriter
home_path = fsrw_class(get_magics_home_path())
home_path.ensure_path_exists()
config_file = fsrw_class(join_paths(home_path.path, CONFIG_JSON))
config_file.ensure_file_exists()
config_text = config_file.read_lines()
line = "".join(config_text).strip()
if line == "":
overrides = {}
else:
overrides = json.loads(line)
override_all(overrides)