def mdf_selected(self, parameter_s=""): """ Return tuples of (ctx, node) for the currently selected nodes in the mdf viewer. %mdf_selected """ return viewer.get_selected()
def _magic_dataframe(self, parameter_s, widepanel=False, single_df=True): """Implementation for magic_dataframe and magic_widepanel""" # the first two arguments are dates, and after that it's a list of nodes # with some optional keyword args, ie %mdf_df <start> <end> node, node, node, shifts=[{x:1}, {x:2}] args = arg_names = tokenize(parameter_s) args = [ _try_eval(x, self.shell.user_global_ns, self.shell.user_ns) for x in args ] args = list(zip(arg_names, args)) start = None if len(args) > 0: arg_name, arg = args.pop(0) start = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns) end = None if len(args) > 0: arg_name, arg = args.pop(0) end = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns) # the final argument can be the number of processes to use num_processes = 0 if len(args) > 0: arg_name, arg = args[-1] if isinstance(arg, basestring) and arg.startswith("||"): arg_name, arg = args.pop() num_processes = int(arg[2:]) # the next to last parameter may be a shift set or list of # shift sets. has_shifts = False shift_sets = [{}] # always have at least one empty shift set shift_names = ["_0"] arg_name, arg = args[-1] if len(args) > 0 else (None, None) if not isinstance(arg, MDFNode): arg_name, arg = args.pop() named_shift_sets = _get_shift_sets(arg_name, arg) if named_shift_sets: shift_names, shift_sets = zip(*named_shift_sets) has_shifts = True # any remaining arguments are the nodes nodes = [] node_var_names = [] for arg_name, node in args: assert isinstance(node, MDFNode), "%s is not a node" % arg_name nodes.append(node) node_var_names.append(arg_name) curr_ctx = _get_current_context() ctxs = [None] * len(nodes) if not nodes: # get the selected nodes from the viewer if _viewer_imported: selected = viewer.get_selected() ctxs, nodes = zip(*selected) for i, (ctx, node) in enumerate(selected): assert ctx.is_shift_of(curr_ctx), \ "selected node '%s' is not in the current context" % node.name # replace any contexts that are simply the current context with None # so that shifting works correctly if ctx is curr_ctx: ctxs[i] = None # if there are shifts then all the contexts have to be None otherwise the # shifts won't work correctly. This could be relaxed later if it causes problems, # but for now this makes the code simpler. if has_shifts: assert np.array([x is None for x in ctxs]).all(), \ "Can't apply shifts when contexts are explicitly specified" # list df_builders, one per node or group of nodes callbacks = [] df_builders = [] if widepanel or not single_df: # build multiple dataframes for node, ctx in zip(nodes, ctxs): if ctx is None: df_builder = DataFrameBuilder([node], filter=True) else: df_builder = DataFrameBuilder([node], contexts=[ctx], filter=True) df_builders.append(df_builder) else: # build a single dataframe if np.array([x is None for x in ctxs]).all(): df_builder = DataFrameBuilder(nodes, filter=True) else: df_builder = DataFrameBuilder(nodes, contexts=ctxs, filter=True) df_builders.append(df_builder) # add all the dataframe builders to the callbacks callbacks.extend(df_builders) root_ctx = curr_ctx.get_parent() or curr_ctx date_range = pd.DateRange(start, end, offset=self.__timestep) # Add a progress bar to the callbacks callbacks.append(ProgressBar(date_range[0], date_range[-1])) shifted_ctxs = run(date_range, callbacks, ctx=root_ctx, shifts=shift_sets, num_processes=num_processes) if not has_shifts: shifted_ctxs = [root_ctx] # when returning a list of results because multiple shifts have been specified # use a named tuple with the items being the names of the shifts tuple_ctr = tuple if has_shifts: # Currying hell yeah tuple_ctr = partial(ShiftedResultsTuple, shift_names) if widepanel: wps = [] for shift_name, shift_set, shifted_ctx in zip( shift_names, shift_sets, shifted_ctxs): wp_dict = {} for node_var_name, df_builder in zip(node_var_names, df_builders): wp_dict[node_var_name] = df_builder.get_dataframe( shifted_ctx) wp = pd.WidePanel.from_dict(wp_dict) if has_shifts: wp = WidePanelWithShiftSet(wp, shift_name, shift_set) wps.append(wp) if len(wps) == 1: return wps[0] return tuple_ctr(*wps) # list a list of lists of dataframes # [[dfs for one shift set], [dfs for next shift set], ...] df_lists = [] for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets, shifted_ctxs): dfs = [] for df_builder in df_builders: df = df_builder.get_dataframe(shifted_ctx) if has_shifts: df = DataFrameWithShiftSet(df, shift_name, shift_set) dfs.append(df) df_lists.append(dfs) if single_df: # flatten into a single list (there should be one dataframe per shift) dfs = reduce(operator.add, df_lists, []) if len(dfs) == 1: return dfs[0] return tuple_ctr(*dfs) if len(df_lists) == 1: return df_lists[0] return tuple_ctr(*df_lists)
def _magic_dataframe(self, parameter_s, widepanel=False, single_df=True): """Implementation for magic_dataframe and magic_widepanel""" # the first two arguments are dates, and after that it's a list of nodes # with some optional keyword args, ie %mdf_df <start> <end> node, node, node, shifts=[{x:1}, {x:2}] args = arg_names = tokenize(parameter_s) args = [_try_eval(x, self.shell.user_global_ns, self.shell.user_ns) for x in args] args = list(zip(arg_names, args)) start = None if len(args) > 0: arg_name, arg = args.pop(0) start = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns) end = None if len(args) > 0: arg_name, arg = args.pop(0) end = _parse_datetime(arg_name, self.shell.user_global_ns, self.shell.user_ns) # the final argument can be the number of processes to use num_processes = 0 if len(args) > 0: arg_name, arg = args[-1] if isinstance(arg, basestring) and arg.startswith("||"): arg_name, arg = args.pop() num_processes = int(arg[2:]) # the next to last parameter may be a shift set or list of # shift sets. has_shifts = False shift_sets = [{}] # always have at least one empty shift set shift_names = ["_0"] arg_name, arg = args[-1] if len(args) > 0 else (None, None) if not isinstance(arg, MDFNode): arg_name, arg = args.pop() named_shift_sets = _get_shift_sets(arg_name, arg) if named_shift_sets: shift_names, shift_sets = zip(*named_shift_sets) has_shifts = True # any remaining arguments are the nodes nodes = [] node_var_names = [] for arg_name, node in args: assert isinstance(node, MDFNode), "%s is not a node" % arg_name nodes.append(node) node_var_names.append(arg_name) curr_ctx = _get_current_context() ctxs = [None] * len(nodes) if not nodes: # get the selected nodes from the viewer if _viewer_imported: selected = viewer.get_selected() ctxs, nodes = zip(*selected) for i, (ctx, node) in enumerate(selected): assert ctx.is_shift_of(curr_ctx), "selected node '%s' is not in the current context" % node.name # replace any contexts that are simply the current context with None # so that shifting works correctly if ctx is curr_ctx: ctxs[i] = None # if there are shifts then all the contexts have to be None otherwise the # shifts won't work correctly. This could be relaxed later if it causes problems, # but for now this makes the code simpler. if has_shifts: assert np.array( [x is None for x in ctxs] ).all(), "Can't apply shifts when contexts are explicitly specified" # list df_builders, one per node or group of nodes callbacks = [] df_builders = [] if widepanel or not single_df: # build multiple dataframes for node, ctx in zip(nodes, ctxs): if ctx is None: df_builder = DataFrameBuilder([node], filter=True) else: df_builder = DataFrameBuilder([node], contexts=[ctx], filter=True) df_builders.append(df_builder) else: # build a single dataframe if np.array([x is None for x in ctxs]).all(): df_builder = DataFrameBuilder(nodes, filter=True) else: df_builder = DataFrameBuilder(nodes, contexts=ctxs, filter=True) df_builders.append(df_builder) # add all the dataframe builders to the callbacks callbacks.extend(df_builders) root_ctx = curr_ctx.get_parent() or curr_ctx date_range = pd.DateRange(start, end, offset=self.__timestep) # Add a progress bar to the callbacks callbacks.append(ProgressBar(date_range[0], date_range[-1])) shifted_ctxs = run(date_range, callbacks, ctx=root_ctx, shifts=shift_sets, num_processes=num_processes) if not has_shifts: shifted_ctxs = [root_ctx] # when returning a list of results because multiple shifts have been specified # use a named tuple with the items being the names of the shifts tuple_ctr = tuple if has_shifts: # Currying hell yeah tuple_ctr = partial(ShiftedResultsTuple, shift_names) if widepanel: wps = [] for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets, shifted_ctxs): wp_dict = {} for node_var_name, df_builder in zip(node_var_names, df_builders): wp_dict[node_var_name] = df_builder.get_dataframe(shifted_ctx) wp = pd.WidePanel.from_dict(wp_dict) if has_shifts: wp = WidePanelWithShiftSet(wp, shift_name, shift_set) wps.append(wp) if len(wps) == 1: return wps[0] return tuple_ctr(*wps) # list a list of lists of dataframes # [[dfs for one shift set], [dfs for next shift set], ...] df_lists = [] for shift_name, shift_set, shifted_ctx in zip(shift_names, shift_sets, shifted_ctxs): dfs = [] for df_builder in df_builders: df = df_builder.get_dataframe(shifted_ctx) if has_shifts: df = DataFrameWithShiftSet(df, shift_name, shift_set) dfs.append(df) df_lists.append(dfs) if single_df: # flatten into a single list (there should be one dataframe per shift) dfs = reduce(operator.add, df_lists, []) if len(dfs) == 1: return dfs[0] return tuple_ctr(*dfs) if len(df_lists) == 1: return df_lists[0] return tuple_ctr(*df_lists)