def group_by_local_date(user_id, from_dt, to_dt, freq, summary_fn_list):
"""
Get grouped data frames for the specified local date range and frequency
:param user_id: id for the user. None for aggregate.
:param from_dt: start local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param to_dt: end local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param freq: since we only expand certain local_dt fields, we can only
support frequencies corresponding to them. These are represented in the
`LocalFreq` enum.
:return: a dict containing the last start_ts of the last section processed
and a result list of ModeStatTimeSummary objects
If there were no matching sections, the last start_ts is None
and the list is empty.
"""
time_query = esttc.TimeComponentQuery("data.start_local_dt", from_dt, to_dt)
section_df = esda.get_data_df(esda.CLEANED_SECTION_KEY,
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return {
"last_ts_processed": None,
"result": [[] for i in range(len(summary_fn_list))]
}
groupby_arr = _get_local_group_by(freq)
time_grouped_df = section_df.groupby(groupby_arr)
local_dt_fill_fn = _get_local_key_to_fill_fn(freq)
return {
"last_ts_processed": section_df.iloc[-1].start_ts,
"result": [grouped_to_summary(time_grouped_df, local_dt_fill_fn, summary_fn)
for summary_fn in summary_fn_list]
}
def group_by_local_date(user_id, from_dt, to_dt, freq, summary_fn_list):
"""
Get grouped data frames for the specified local date range and frequency
:param user_id: id for the user. None for aggregate.
:param from_dt: start local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param to_dt: end local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param freq: since we only expand certain local_dt fields, we can only
support frequencies corresponding to them. These are represented in the
`LocalFreq` enum.
:return: a dict containing the last start_ts of the last section processed
and a result list of ModeStatTimeSummary objects
If there were no matching sections, the last start_ts is None
and the list is empty.
"""
time_query = esttc.TimeComponentQuery("data.start_local_dt", from_dt, to_dt)
section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return {
"last_ts_processed": None,
"result": [[] for i in range(len(summary_fn_list))]
}
groupby_arr = _get_local_group_by(freq)
time_grouped_df = section_df.groupby(groupby_arr)
local_dt_fill_fn = _get_local_key_to_fill_fn(freq)
return {
"last_ts_processed": section_df.iloc[-1].start_ts,
"result": [grouped_to_summary(time_grouped_df, local_dt_fill_fn, summary_fn)
for summary_fn in summary_fn_list]
}
def group_by_timestamp(user_id, start_ts, end_ts, freq, summary_fn):
"""
Get grouped dataframes for the specific time range and at the specified frequency
:param user_id: The user for whom we are computing this information. None for all users.
:param from_ld: The start timestamp
:param to_ld: The end timestamp
:param freq: The frequency as specified in a pandas date_range frequency string.
We only support frequencies of a day or longer in order to return the data
in a format that makes sense
http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
The canonical list can be found at:
> pandas.tseries.offsets.prefix_mapping
:return: a list of ModeStatTimeSummary objects
"""
time_query = estt.TimeQuery("data.start_ts", start_ts, end_ts)
section_df = esda.get_data_df(esda.CLEANED_SECTION_KEY,
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return []
logging.debug("first row is %s" % section_df.iloc[0])
secs_to_nanos = lambda x: x * 10 ** 9
section_df['start_dt'] = pd.to_datetime(secs_to_nanos(section_df.start_ts))
time_grouped_df = section_df.groupby(pd.Grouper(freq=freq, key='start_dt'))
return grouped_to_summary(time_grouped_df, timestamp_fill_times, summary_fn)
def uuid_list_query(modes, time_query, region):
if region is None:
geo_query = None
else:
geo_query = estg.GeoQuery(["data.loc"], region)
extra_query_list = []
if modes is not None:
mode_enum_list = [ecwm.MotionTypes[mode] for mode in modes]
extra_query_list.append(esdlq.get_mode_query(mode_enum_list))
loc_entry_df = esda.get_data_df(esda.CLEANED_LOCATION_KEY, user_id=None,
time_query=time_query, geo_query=geo_query,
extra_query_list=extra_query_list)
if len(loc_entry_df) == 0:
logging.info("No points found matching query, returning empty list")
return []
unique_uuid_list = loc_entry_df.user_id.unique().tolist()
logging.info("Found %d points with %d unique uuids" % (len(loc_entry_df), len(unique_uuid_list)))
return unique_uuid_list
def group_by_timestamp(user_id, start_ts, end_ts, freq, summary_fn_list):
"""
Get grouped dataframes for the specific time range and at the specified frequency
:param user_id: The user for whom we are computing this information. None for all users.
:param from_ld: The start timestamp
:param to_ld: The end timestamp
:param freq: The frequency as specified in a pandas date_range frequency string.
We only support frequencies of a day or longer in order to return the data
in a format that makes sense
http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
The canonical list can be found at:
> pandas.tseries.offsets.prefix_mapping
:return: a dict containing the last start_ts of the last section processed
and a result list of ModeStatTimeSummary objects
If there were no matching sections, the last start_ts is None
and the list is empty.
"""
time_query = estt.TimeQuery("data.start_ts", start_ts, end_ts)
section_df = esda.get_data_df(esda.CLEANED_SECTION_KEY,
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return {
"last_ts_processed": None,
"result": [[] for i in range(len(summary_fn_list))]
}
logging.debug("first row is %s" % section_df.iloc[0])
secs_to_nanos = lambda x: x * 10 ** 9
section_df['start_dt'] = pd.to_datetime(secs_to_nanos(section_df.start_ts))
time_grouped_df = section_df.groupby(pd.Grouper(freq=freq, key='start_dt'))
return {
"last_ts_processed": section_df.iloc[-1].start_ts,
"result": [grouped_to_summary(time_grouped_df, timestamp_fill_times, summary_fn)
for summary_fn in summary_fn_list]
}
def group_by_timestamp(user_id, start_ts, end_ts, freq, summary_fn_list):
"""
Get grouped dataframes for the specific time range and at the specified frequency
:param user_id: The user for whom we are computing this information. None for all users.
:param from_ld: The start timestamp
:param to_ld: The end timestamp
:param freq: The frequency as specified in a pandas date_range frequency string.
We only support frequencies of a day or longer in order to return the data
in a format that makes sense
http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
The canonical list can be found at:
> pandas.tseries.offsets.prefix_mapping
:return: a dict containing the last start_ts of the last section processed
and a result list of ModeStatTimeSummary objects
If there were no matching sections, the last start_ts is None
and the list is empty.
"""
time_query = estt.TimeQuery("data.start_ts", start_ts, end_ts)
section_df = esda.get_data_df(eac.get_section_key_for_analysis_results(),
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return {
"last_ts_processed": None,
"result": [[] for i in range(len(summary_fn_list))]
}
logging.debug("first row is %s" % section_df.iloc[0])
secs_to_nanos = lambda x: x * 10 ** 9
section_df['start_dt'] = pd.to_datetime(secs_to_nanos(section_df.start_ts))
time_grouped_df = section_df.groupby(pd.Grouper(freq=freq, key='start_dt'))
return {
"last_ts_processed": section_df.iloc[-1].start_ts,
"result": [grouped_to_summary(time_grouped_df, timestamp_fill_times, summary_fn)
for summary_fn in summary_fn_list]
}
def uuid_list_query(modes, time_query, region):
if region is None:
geo_query = None
else:
geo_query = estg.GeoQuery(["data.loc"], region)
extra_query_list = []
if modes is not None:
mode_enum_list = [ecwm.MotionTypes[mode] for mode in modes]
extra_query_list.append(esdlq.get_mode_query(mode_enum_list))
loc_entry_df = esda.get_data_df(esda.CLEANED_LOCATION_KEY,
user_id=None,
time_query=time_query,
geo_query=geo_query,
extra_query_list=extra_query_list)
if len(loc_entry_df) == 0:
logging.info("No points found matching query, returning empty list")
return []
unique_uuid_list = loc_entry_df.user_id.unique().tolist()
logging.info("Found %d points with %d unique uuids" %
(len(loc_entry_df), len(unique_uuid_list)))
return unique_uuid_list
def group_by_local_date(user_id, from_dt, to_dt, freq, summary_fn):
"""
Get grouped data frames for the specified local date range and frequency
:param user_id: id for the user. None for aggregate.
:param from_dt: start local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param to_dt: end local dt object. We assume that only the year, month
and date entries are filled in and represent a date range.
:param freq: since we only expand certain local_dt fields, we can only
support frequencies corresponding to them. These are represented in the
`LocalFreq` enum.
:return: pandas.core.groupby.DataFrameGroupBy object
"""
time_query = esttc.TimeComponentQuery("data.start_local_dt", from_dt, to_dt)
section_df = esda.get_data_df(esda.CLEANED_SECTION_KEY,
user_id=user_id, time_query=time_query,
geo_query=None)
if len(section_df) == 0:
logging.info("Found no entries for user %s, time_query %s" % (user_id, time_query))
return []
groupby_arr = _get_local_group_by(freq)
time_grouped_df = section_df.groupby(groupby_arr)
local_dt_fill_fn = _get_local_key_to_fill_fn(freq)
return grouped_to_summary(time_grouped_df, local_dt_fill_fn, summary_fn)
