def search(self,
df,
num_examples_per_instance,
minimum_data=None,
gap=None,
drop_empty=True,
label_type=None,
verbose=True,
*args,
**kwargs):
"""Searches the data to calculates labels.
Args:
df (DataFrame): Data frame to search and extract labels.
num_examples_per_instance (int or dict): The expected number of examples to return from each entity group.
A dictionary can be used to further specify the expected number of examples to return from each label.
minimum_data (str): Minimum data before starting search. Default value is first time of index.
gap (str or int): Time between examples. Default value is window size.
If an integer, search will start on the first event after the minimum data.
drop_empty (bool): Whether to drop empty slices. Default value is True.
label_type (str): The label type can be "continuous" or "categorical". Default value is the inferred label type.
verbose (bool): Whether to render progress bar. Default value is True.
*args: Positional arguments for labeling function.
**kwargs: Keyword arguments for labeling function.
Returns:
lt (LabelTimes): Calculated labels with cutoff times.
"""
assert self.labeling_function, 'missing labeling function(s)'
self._check_example_count(num_examples_per_instance, gap)
self.window_size = self.window_size or len(df)
gap = to_offset(gap or self.window_size)
is_label_search = isinstance(num_examples_per_instance, dict)
search = (LabelSearch if is_label_search else ExampleSearch)(num_examples_per_instance)
records = self._run_search(
df=df,
search=search,
gap=gap,
min_data=minimum_data,
drop_empty=drop_empty,
verbose=verbose,
*args,
**kwargs,
)
lt = LabelTimes(
data=records,
target_columns=list(self.labeling_function),
target_entity=self.target_entity,
search_settings={
'num_examples_per_instance': num_examples_per_instance,
'minimum_data': str(minimum_data),
'window_size': str(self.window_size),
'gap': str(gap),
},
)
return lt
def slice(self, df, num_examples_per_instance, minimum_data=None, gap=None, drop_empty=True, verbose=False):
"""Generates data slices of target entity.
Args:
df (DataFrame): Data frame to create slices on.
num_examples_per_instance (int): Number of examples per unique instance of target entity.
minimum_data (str): Minimum data before starting search. Default value is first time of index.
gap (str or int): Time between examples. Default value is window size.
If an integer, search will start on the first event after the minimum data.
drop_empty (bool): Whether to drop empty slices. Default value is True.
verbose (bool): Whether to print metadata about slice. Default value is False.
Returns:
ds (generator): Returns a generator of data slices.
"""
self._check_example_count(num_examples_per_instance, gap)
self.window_size = self.window_size or len(df)
gap = to_offset(gap or self.window_size)
groups = self.set_index(df).groupby(self.target_entity)
if num_examples_per_instance == -1:
num_examples_per_instance = float('inf')
for key, df in groups:
slices = self._slice(df=df, gap=gap, min_data=minimum_data, drop_empty=drop_empty)
for ds in slices:
ds.context.target_instance = key
if verbose: print(ds)
yield ds
if ds.context.slice_number >= num_examples_per_instance:
break
def _set_window_size(self, window_size):
"""Set and format initial window size parameter.
Args:
window_size (str or int): Duration of each data slice.
The default value for window size is all future data.
"""
if window_size is not None:
window_size = to_offset(window_size)
self.window_size = window_size
def slice(self,
df,
num_examples_per_instance,
minimum_data=None,
gap=None,
drop_empty=True,
verbose=False):
"""Generates data slices of target entity.
Args:
df (DataFrame) : Data frame to create slices on.
num_examples_per_instance (int) : Number of examples per unique instance of target entity.
minimum_data (str) : Minimum data before starting search. Default value is first time of index.
gap (str or int) : Time between examples. Default value is window size.
If an integer, search will start on the first event after the minimum data.
drop_empty (bool) : Whether to drop empty slices. Default value is True.
verbose (bool) : Whether to print metadata about slice. Default value is False.
Returns:
DataSlice : Returns data slice.
"""
if self.window_size is None and gap is None:
more_than_one = num_examples_per_instance > 1
assert not more_than_one, "must specify gap if num_examples > 1 and window size = none"
self.window_size = self.window_size or len(df)
gap = to_offset(gap or self.window_size)
df = self.set_index(df)
if num_examples_per_instance == -1:
num_examples_per_instance = float('inf')
for group in df.groupby(self.target_entity):
slices = self._get_slices(group=group,
gap=gap,
min_data=minimum_data,
drop_empty=drop_empty)
for df in slices:
if verbose:
print(df)
yield df
if df.context.slice_number >= num_examples_per_instance:
break
def __init__(self,
target_entity,
time_index,
labeling_function,
window_size=None,
label_type=None):
"""Creates an instance of label maker.
Args:
target_entity (str) : Entity on which to make labels.
time_index (str): Name of time column in the data frame.
labeling_function (function) : Function that transforms a data slice to a label.
window_size (str or int) : Duration of each data slice.
The default value for window size is all future data.
"""
self.target_entity = target_entity
self.time_index = time_index
self.labeling_function = labeling_function
self.window_size = window_size
if self.window_size is not None:
self.window_size = to_offset(self.window_size)
def _get_slices(self, group, gap=None, min_data=None, drop_empty=True):
"""Generate data slices for group.
Args:
df (DataFrame) : Data frame to generate data slices.
gap (str or int) : Time between examples. Default value is window size.
If an integer, search will start on the first event after the minimum data.
min_data (int or str or Timestamp) : Threshold to cutoff data.
drop_empty (bool) : Whether to drop empty slices. Default value is True.
Returns:
DataSlice : Returns a data slice.
"""
key, df = group
self.window_size = self.window_size or len(df)
gap = to_offset(gap or self.window_size)
df = df.loc[df.index.notnull()]
assert df.index.is_monotonic_increasing, "Please sort your dataframe chronologically before calling search"
if df.empty:
return
threshold = min_data or df.index[0]
df, cutoff_time = cutoff_data(df=df, threshold=threshold)
if df.empty:
return
if isinstance(gap, int):
cutoff_time = df.index[0]
df = DataSlice(df)
df.context = Context(slice_number=0,
target_entity=self.target_entity,
target_instance=key)
def iloc(index, i):
if i < index.size:
return index[i]
while not df.empty and cutoff_time <= df.index[-1]:
if isinstance(self.window_size, int):
df_slice = df.iloc[:self.window_size]
window_end = iloc(df.index, self.window_size)
else:
window_end = cutoff_time + self.window_size
df_slice = df[:window_end]
# Pandas includes both endpoints when slicing by time.
# This results in the right endpoint overlapping in consecutive data slices.
# Resolved by making the right endpoint exclusive.
# https://pandas.pydata.org/pandas-docs/version/0.19/gotchas.html#endpoints-are-inclusive
if not df_slice.empty:
is_overlap = df_slice.index == window_end
if df_slice.index.size > 1 and is_overlap.any():
df_slice = df_slice[~is_overlap]
df_slice.context.window = (cutoff_time, window_end)
if isinstance(gap, int):
gap_end = iloc(df.index, gap)
df_slice.context.gap = (cutoff_time, gap_end)
df = df.iloc[gap:]
if not df.empty:
cutoff_time = df.index[0]
else:
gap_end = cutoff_time + gap
df_slice.context.gap = (cutoff_time, gap_end)
cutoff_time += gap
if cutoff_time <= df.index[-1]:
df = df[cutoff_time:]
if df_slice.empty and drop_empty:
continue
df.context.slice_number += 1
yield df_slice