def get_crop(self, date_times, batch_size):
"""
:param date_time:
:return:(batch,seq_len,h,w,1)
"""
assert date_times is not None
frame_dat = np.zeros((batch_size, cfg.IN_SEQ + cfg.OUT_SEQ,
cfg.H_TRAIN, cfg.W_TRAIN, 1),
dtype=np.uint8)
for i in range(len(date_times)):
in_dir = os.path.join(cfg.CROP_DATA_PATH,
date_times[i].strftime("%Y%m%d%H%M"), 'in',
'1')
in_paths = [
os.path.join(in_dir,
str(i) + '.png')
for i in range(1 + cfg.IN_SEQ, 5 + cfg.IN_SEQ + 1)
]
out_dir = os.path.join(cfg.CROP_DATA_PATH,
date_times[i].strftime("%Y%m%d%H%M"), 'out',
'1')
out_paths = [
os.path.join(out_dir,
str(i) + '.png')
for i in range(1, cfg.OUT_SEQ + 1)
]
frame_dat[i, 0:cfg.IN_SEQ] = quick_read_frames(
in_paths, cfg.H_TRAIN, cfg.W_TRAIN)
frame_dat[i,
cfg.IN_SEQ:cfg.OUT_SEQ + cfg.IN_SEQ] = quick_read_frames(
out_paths, cfg.H_TRAIN, cfg.W_TRAIN)
return frame_dat
def get_test_data(self, datetimes, batch_size):
# frame_dats=np.zeros(batch_size,cfg.IN_SEQ+cfg.OUT_SEQ,cfg.PRED_H,cfg.PRED_W,cfg.IN_CHANEL)
frame_dats = []
for datetime in datetimes:
print(datetime)
datetime_clip1 = pd.date_range(end=datetime,
periods=cfg.DISPLAY_IN_SEQ,
freq=self._base_freq)
datetime_clip2 = pd.date_range(start=datetime,
periods=cfg.OUT_SEQ + 1,
freq=self._base_freq)
date_time_clip = []
date_time_clip[0:cfg.DISPLAY_IN_SEQ] = datetime_clip1
date_time_clip[cfg.DISPLAY_IN_SEQ:cfg.OUT_SEQ +
cfg.DISPLAY_IN_SEQ] = datetime_clip2[1:]
paths = []
for i in range(len(date_time_clip)):
path = convert_datetime_to_filepath(date_time_clip[i])
print('collect path: ---' + path)
paths.append(path)
try:
frame_dat = quick_read_frames(paths, 720, 900)
except IOError:
continue
frame_dats.append(frame_dat)
frame_dats = np.array(frame_dats)
# test_data=get_480x480_clip(clip=frame_dat,mode='pred')
return frame_dats.reshape(
[batch_size, cfg.DISPLAY_IN_SEQ + cfg.OUT_SEQ, 720, 900, 1])
def crop_dataset(start,end,in_len,group_size):
datetime_clip = pd.date_range(start=start,
end=end,
freq='6T')
for i in range(in_len-1,len(datetime_clip)):
# print(i)
paths = []
for j in range(i-c.IN_SEQ+1,i+1+c.OUT_SEQ):
if j>=len(datetime_clip):
break
path = convert_datetime_to_filepath(datetime_clip[j])
if not os.path.exists(path):
break
paths.append(path)
if len(paths)!=c.IN_SEQ+c.OUT_SEQ:
print(datetime_clip[i].strftime("%Y%m%d%H%M") + ' Miss')
continue
train_data = quick_read_frames(paths,700, 900)
crop_data=get_480x480_clip(clip=train_data,size=group_size)
if crop_data is None:
print(datetime_clip[i].strftime("%Y%m%d%H%M")+' Too little pixels')
continue
dir = os.path.join(c.CROP_DATA_PATH,datetime_clip[i].strftime("%Y%m%d%H%M"))
if not os.path.exists(dir):
os.makedirs(dir)
for k in range(0,group_size):
save_in_dir=os.path.join(dir,'in',str(k+1))
save_out_dir=os.path.join(dir,'out',str(k+1))
if not os.path.exists(save_in_dir):
os.makedirs(save_in_dir)
if not os.path.exists(save_out_dir):
os.makedirs(save_out_dir)
save_crop_data(save_in_dir,save_out_dir, crop_data[k])
def _load_frames(self, datetime_clips):
assert isinstance(datetime_clips, list)
for clip in datetime_clips:
assert len(clip) == self._seq_len
batch_size = len(datetime_clips)
frame_dat = np.zeros(
(batch_size, self._seq_len, self._height, self._width, 1),
dtype=np.uint8)
if self.sample_mode == "random":
paths = []
hit_inds = []
miss_inds = []
for i in range(self._seq_len):
for j in range(batch_size):
timestamp = datetime_clips[j][i]
if timestamp in self._df_index_set:
paths.append(
convert_datetime_to_filepath(datetime_clips[j][i]))
hit_inds.append([i, j])
else:
miss_inds.append([i, j])
hit_inds = np.array(hit_inds, dtype=np.int)
all_frame_dat = quick_read_frames(paths, self._height, self._width)
frame_dat[hit_inds[:, 1], hit_inds[:, 0], :, :, :] = all_frame_dat
# frame_dat=get_480x480_clip(frame_dat)
else:
# Get the first_timestamp and the last_timestamp in the datetime_clips
first_timestamp = datetime_clips[-1][-1]
last_timestamp = datetime_clips[0][0]
for i in range(self._seq_len):
for j in range(batch_size):
timestamp = datetime_clips[j][i]
if timestamp in self._df_index_set:
first_timestamp = min(first_timestamp, timestamp)
last_timestamp = max(last_timestamp, timestamp)
if self._buffer_datetime_keys is None or \
not (first_timestamp in self._buffer_datetime_keys
and last_timestamp in self._buffer_datetime_keys):
read_begin_ind = self._df.index.get_loc(first_timestamp)
read_end_ind = self._df.index.get_loc(last_timestamp) + 1
read_end_ind = min(
read_begin_ind + self._buffer_mult *
(read_end_ind - read_begin_ind), self._df.size)
self._buffer_datetime_keys = self._df.index[
read_begin_ind:read_end_ind]
# Fill in the buffer
paths = []
for i in range(self._buffer_datetime_keys.size):
paths.append(
convert_datetime_to_filepath(
self._buffer_datetime_keys[i]))
self._buffer_frame_dat = quick_read_frames(
paths, self._height, self._width)
for i in range(self._seq_len):
for j in range(batch_size):
timestamp = datetime_clips[j][i]
if timestamp in self._df_index_set:
assert timestamp in self._buffer_datetime_keys
ind = self._buffer_datetime_keys.get_loc(timestamp)
frame_dat[j, i, :, :, :] = self._buffer_frame_dat[
ind, :, :, :]
frame_dat = get_480x480_clip(frame_dat)
return frame_dat