def get_data(self, time_from: dt.datetime = None, time_to: dt.datetime = None, include_from=True, include_to=True,
read_func=None, empty=None, **kwargs):
table = self.settings.TABLES[kwargs[self.settings.TABLE]]
tz, file_structure = table.TIMEZONE, table.FILE_STRUCTURE
time_from = to_tz_datetime(pd.to_datetime(time_from), to_tz=tz)
time_to = to_tz_datetime(pd.to_datetime(time_to), to_tz=tz)
date_from = None if time_from is None else time_from.normalize()
date_to = None if time_to is None else time_to.normalize()
read_func = table.read_func() if read_func is None else read_func
def from_files(files, client):
yield pd.DataFrame()
for fpath in files:
with client.open(fpath) as fhandle:
yield read_func(fhandle)
with self.transport_session() as transport:
with paramiko.SFTPClient.from_transport(transport) as sftp:
files_found = self.find_files(file_structure, sftp, False, **kwargs)
dated_fpaths = pd.DataFrame((table.date_from_filename(x), x) for x in files_found)
dated_fpaths = dated_fpaths.set_index(0).sort_index().loc[date_from: date_to, 1]
data_df = pd.concat(from_files(dated_fpaths, sftp))
if data_df.empty:
return empty
results = pd.DataFrame()
for _, tag_df in data_df.groupby(table.Tags.values()):
i, j = bound_indices(tag_df.index,
lambda x: isin_closed(x, time_from, time_to, (include_from, include_to)))
results = results.append(tag_df.iloc[i:j])
return results
def date_mapping(self, a_date: datetime, date_map: dict):
weekday = a_date.weekday()
week_start = a_date.normalize() - timedelta(days=weekday)
weekday = timedelta(weekday)
diff = date_map[weekday]
diff = timedelta(days=diff)
return week_start + diff
