def mdf_selected(self, parameter_s=""):
"""
Return tuples of (ctx, node) for the currently selected nodes in
the mdf viewer.
%mdf_selected
"""
return viewer.get_selected()
def mdf_selected(self, parameter_s=""):
"""
Return tuples of (ctx, node) for the currently selected nodes in
the mdf viewer.
%mdf_selected
"""
return viewer.get_selected()
def _magic_dataframe(self, parameter_s, widepanel=False, single_df=True):
"""Implementation for magic_dataframe and magic_widepanel"""
# the first two arguments are dates, and after that it's a list of nodes
# with some optional keyword args, ie %mdf_df <start> <end> node, node, node, shifts=[{x:1}, {x:2}]
args = arg_names = tokenize(parameter_s)
args = [
_try_eval(x, self.shell.user_global_ns, self.shell.user_ns)
for x in args
]
args = list(zip(arg_names, args))
start = None
if len(args) > 0:
arg_name, arg = args.pop(0)
start = _parse_datetime(arg_name, self.shell.user_global_ns,
self.shell.user_ns)
end = None
if len(args) > 0:
arg_name, arg = args.pop(0)
end = _parse_datetime(arg_name, self.shell.user_global_ns,
self.shell.user_ns)
# the final argument can be the number of processes to use
num_processes = 0
if len(args) > 0:
arg_name, arg = args[-1]
if isinstance(arg, basestring) and arg.startswith("||"):
arg_name, arg = args.pop()
num_processes = int(arg[2:])
# the next to last parameter may be a shift set or list of
# shift sets.
has_shifts = False
shift_sets = [{}] # always have at least one empty shift set
shift_names = ["_0"]
arg_name, arg = args[-1] if len(args) > 0 else (None, None)
if not isinstance(arg, MDFNode):
arg_name, arg = args.pop()
named_shift_sets = _get_shift_sets(arg_name, arg)
if named_shift_sets:
shift_names, shift_sets = zip(*named_shift_sets)
has_shifts = True
# any remaining arguments are the nodes
nodes = []
node_var_names = []
for arg_name, node in args:
assert isinstance(node, MDFNode), "%s is not a node" % arg_name
nodes.append(node)
node_var_names.append(arg_name)
curr_ctx = _get_current_context()
ctxs = [None] * len(nodes)
if not nodes:
# get the selected nodes from the viewer
if _viewer_imported:
selected = viewer.get_selected()
ctxs, nodes = zip(*selected)
for i, (ctx, node) in enumerate(selected):
assert ctx.is_shift_of(curr_ctx), \
"selected node '%s' is not in the current context" % node.name
# replace any contexts that are simply the current context with None
# so that shifting works correctly
if ctx is curr_ctx:
ctxs[i] = None
# if there are shifts then all the contexts have to be None otherwise the
# shifts won't work correctly. This could be relaxed later if it causes problems,
# but for now this makes the code simpler.
if has_shifts:
assert np.array([x is None for x in ctxs]).all(), \
"Can't apply shifts when contexts are explicitly specified"
# list df_builders, one per node or group of nodes
callbacks = []
df_builders = []
if widepanel or not single_df:
# build multiple dataframes
for node, ctx in zip(nodes, ctxs):
if ctx is None:
df_builder = DataFrameBuilder([node], filter=True)
else:
df_builder = DataFrameBuilder([node],
contexts=[ctx],
filter=True)
df_builders.append(df_builder)
else:
# build a single dataframe
if np.array([x is None for x in ctxs]).all():
df_builder = DataFrameBuilder(nodes, filter=True)
else:
df_builder = DataFrameBuilder(nodes,
contexts=ctxs,
filter=True)
df_builders.append(df_builder)
# add all the dataframe builders to the callbacks
callbacks.extend(df_builders)
root_ctx = curr_ctx.get_parent() or curr_ctx
date_range = pd.DateRange(start, end, offset=self.__timestep)
# Add a progress bar to the callbacks
callbacks.append(ProgressBar(date_range[0], date_range[-1]))
shifted_ctxs = run(date_range,
callbacks,
ctx=root_ctx,
shifts=shift_sets,
num_processes=num_processes)
if not has_shifts:
shifted_ctxs = [root_ctx]
# when returning a list of results because multiple shifts have been specified
# use a named tuple with the items being the names of the shifts
tuple_ctr = tuple
if has_shifts:
# Currying hell yeah
tuple_ctr = partial(ShiftedResultsTuple, shift_names)
if widepanel:
wps = []
for shift_name, shift_set, shifted_ctx in zip(
shift_names, shift_sets, shifted_ctxs):
wp_dict = {}
for node_var_name, df_builder in zip(node_var_names,
df_builders):
wp_dict[node_var_name] = df_builder.get_dataframe(
shifted_ctx)
wp = pd.WidePanel.from_dict(wp_dict)
if has_shifts:
wp = WidePanelWithShiftSet(wp, shift_name, shift_set)
wps.append(wp)
if len(wps) == 1:
return wps[0]
return tuple_ctr(*wps)
# list a list of lists of dataframes
# [[dfs for one shift set], [dfs for next shift set], ...]
df_lists = []
for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets,
shifted_ctxs):
dfs = []
for df_builder in df_builders:
df = df_builder.get_dataframe(shifted_ctx)
if has_shifts:
df = DataFrameWithShiftSet(df, shift_name, shift_set)
dfs.append(df)
df_lists.append(dfs)
if single_df:
# flatten into a single list (there should be one dataframe per shift)
dfs = reduce(operator.add, df_lists, [])
if len(dfs) == 1:
return dfs[0]
return tuple_ctr(*dfs)
if len(df_lists) == 1:
return df_lists[0]
return tuple_ctr(*df_lists)
def _magic_dataframe(self, parameter_s, widepanel=False, single_df=True):
"""Implementation for magic_dataframe and magic_widepanel"""
# the first two arguments are dates, and after that it's a list of nodes
# with some optional keyword args, ie %mdf_df <start> <end> node, node, node, shifts=[{x:1}, {x:2}]
args = arg_names = tokenize(parameter_s)
args = [_try_eval(x, self.shell.user_global_ns, self.shell.user_ns) for x in args]
args = list(zip(arg_names, args))
start = None
if len(args) > 0:
arg_name, arg = args.pop(0)
start = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns)
end = None
if len(args) > 0:
arg_name, arg = args.pop(0)
end = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns)
# the final argument can be the number of processes to use
num_processes = 0
if len(args) > 0:
arg_name, arg = args[-1]
if isinstance(arg, basestring) and arg.startswith("||"):
arg_name, arg = args.pop()
num_processes = int(arg[2:])
# the next to last parameter may be a shift set or list of
# shift sets.
has_shifts = False
shift_sets = [{}] # always have at least one empty shift set
shift_names = ["_0"]
arg_name, arg = args[-1] if len(args) > 0 else (None, None)
if not isinstance(arg, MDFNode):
arg_name, arg = args.pop()
named_shift_sets = _get_shift_sets(arg_name, arg)
if named_shift_sets:
shift_names, shift_sets = zip(*named_shift_sets)
has_shifts = True
# any remaining arguments are the nodes
nodes = []
node_var_names = []
for arg_name, node in args:
assert isinstance(node, MDFNode), "%s is not a node" % arg_name
nodes.append(node)
node_var_names.append(arg_name)
curr_ctx = _get_current_context()
ctxs = [None] * len(nodes)
if not nodes:
# get the selected nodes from the viewer
if _viewer_imported:
selected = viewer.get_selected()
ctxs, nodes = zip(*selected)
for i, (ctx, node) in enumerate(selected):
assert ctx.is_shift_of(curr_ctx), "selected node '%s' is not in the current context" % node.name
# replace any contexts that are simply the current context with None
# so that shifting works correctly
if ctx is curr_ctx:
ctxs[i] = None
# if there are shifts then all the contexts have to be None otherwise the
# shifts won't work correctly. This could be relaxed later if it causes problems,
# but for now this makes the code simpler.
if has_shifts:
assert np.array(
[x is None for x in ctxs]
).all(), "Can't apply shifts when contexts are explicitly specified"
# list df_builders, one per node or group of nodes
callbacks = []
df_builders = []
if widepanel or not single_df:
# build multiple dataframes
for node, ctx in zip(nodes, ctxs):
if ctx is None:
df_builder = DataFrameBuilder([node], filter=True)
else:
df_builder = DataFrameBuilder([node], contexts=[ctx], filter=True)
df_builders.append(df_builder)
else:
# build a single dataframe
if np.array([x is None for x in ctxs]).all():
df_builder = DataFrameBuilder(nodes, filter=True)
else:
df_builder = DataFrameBuilder(nodes, contexts=ctxs, filter=True)
df_builders.append(df_builder)
# add all the dataframe builders to the callbacks
callbacks.extend(df_builders)
root_ctx = curr_ctx.get_parent() or curr_ctx
date_range = pd.DateRange(start, end, offset=self.__timestep)
# Add a progress bar to the callbacks
callbacks.append(ProgressBar(date_range[0], date_range[-1]))
shifted_ctxs = run(date_range, callbacks, ctx=root_ctx, shifts=shift_sets, num_processes=num_processes)
if not has_shifts:
shifted_ctxs = [root_ctx]
# when returning a list of results because multiple shifts have been specified
# use a named tuple with the items being the names of the shifts
tuple_ctr = tuple
if has_shifts:
# Currying hell yeah
tuple_ctr = partial(ShiftedResultsTuple, shift_names)
if widepanel:
wps = []
for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets, shifted_ctxs):
wp_dict = {}
for node_var_name, df_builder in zip(node_var_names, df_builders):
wp_dict[node_var_name] = df_builder.get_dataframe(shifted_ctx)
wp = pd.WidePanel.from_dict(wp_dict)
if has_shifts:
wp = WidePanelWithShiftSet(wp, shift_name, shift_set)
wps.append(wp)
if len(wps) == 1:
return wps[0]
return tuple_ctr(*wps)
# list a list of lists of dataframes
# [[dfs for one shift set], [dfs for next shift set], ...]
df_lists = []
for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets, shifted_ctxs):
dfs = []
for df_builder in df_builders:
df = df_builder.get_dataframe(shifted_ctx)
if has_shifts:
df = DataFrameWithShiftSet(df, shift_name, shift_set)
dfs.append(df)
df_lists.append(dfs)
if single_df:
# flatten into a single list (there should be one dataframe per shift)
dfs = reduce(operator.add, df_lists, [])
if len(dfs) == 1:
return dfs[0]
return tuple_ctr(*dfs)
if len(df_lists) == 1:
return df_lists[0]
return tuple_ctr(*df_lists)
