def get_data(self, time_from: dt.datetime = None, time_to: dt.datetime = None, include_from=True, include_to=True, read_func=None, empty=None, **kwargs): table = self.settings.TABLES[kwargs[self.settings.TABLE]] tz, file_structure = table.TIMEZONE, table.FILE_STRUCTURE time_from = to_tz_datetime(pd.to_datetime(time_from), to_tz=tz) time_to = to_tz_datetime(pd.to_datetime(time_to), to_tz=tz) date_from = None if time_from is None else time_from.normalize() date_to = None if time_to is None else time_to.normalize() read_func = table.read_func() if read_func is None else read_func def from_files(files, client): yield pd.DataFrame() for fpath in files: with client.open(fpath) as fhandle: yield read_func(fhandle) with self.transport_session() as transport: with paramiko.SFTPClient.from_transport(transport) as sftp: files_found = self.find_files(file_structure, sftp, False, **kwargs) dated_fpaths = pd.DataFrame((table.date_from_filename(x), x) for x in files_found) dated_fpaths = dated_fpaths.set_index(0).sort_index().loc[date_from: date_to, 1] data_df = pd.concat(from_files(dated_fpaths, sftp)) if data_df.empty: return empty results = pd.DataFrame() for _, tag_df in data_df.groupby(table.Tags.values()): i, j = bound_indices(tag_df.index, lambda x: isin_closed(x, time_from, time_to, (include_from, include_to))) results = results.append(tag_df.iloc[i:j]) return results
def date_mapping(self, a_date: datetime, date_map: dict): weekday = a_date.weekday() week_start = a_date.normalize() - timedelta(days=weekday) weekday = timedelta(weekday) diff = date_map[weekday] diff = timedelta(days=diff) return week_start + diff