def get_csv_file_list(directorys, file_selector):
if type(directorys) is tuple:
return [
PathList.match(matchCsv, *file_selector)(directory)
for directory in directorys
]
else:
return [PathList.match(matchCsv, *file_selector)(directorys)]
def copy_hdr(self):
path = PathList.match(r"\.hdr$")(
self.read_option.read_directory).files()[0]
shutil.copy(
path,
os.path.join(self.read_option.output_directory,
os.path.basename(path)))
def read_original_dat(meta: VectorReadOption,
common,
sep=",",
preprocess=[timeShifter]):
files = PathList.match(r"\.dat$")(meta.read_directory).files(verbose=True)
dfs = []
for file in files:
reader = pd.read_csv(file,
sep=sep,
chunksize=50000,
names=common["columns"],
header=meta.header_row)
df = pd.concat(
pip(*[f(meta, common) for f in preprocess])(r)
for r in tqdm(reader))
display(df.head(10))
dfs.append(df)
df = pd.concat(dfs)
df_droped = df.drop(common["drop"], axis=1)
return df_droped
def read_sen(self):
path = PathList.match(r"\.sen\.csv$")(
self.read_option.output_directory).files()[0]
sen = self.Reader.create()\
.setPath(path)\
.read(0)\
.assemble(
toDateTime,
dataframe.setTimeSeriesIndex("datetime")
).df
return sen
def read_vhd(self):
path = PathList.match(r"\.vhd\.csv$")(
self.read_option.output_directory).files()[0]
vhd = self.Reader.create()\
.setPath(path)\
.read(0)\
.assemble(
toDateTime,
dataframe.setTimeSeriesIndex("datetime")
).df
# バースト時刻の差 (min)
delta_burst_time = [0]
vhd_index = vhd.index
for i in range(1, len(vhd_index)):
time_delta = vhd_index[i] - vhd_index[i - 1]
delta_burst_time.append(time_delta.seconds / 60)
vhd = vhd.assign(delta_burst_time=delta_burst_time)
return vhd
def read_dat(self):
dat = pd.read_csv(PathList.match(r"dat\.csv")(
self.read_option.output_directory).files(True)[0],
index_col=0,
parse_dates=[0])
return dat
def convert_sen(self):
VectorConverter.convert_csv(
PathList.match(r"\.sen$")(
self.read_option.read_directory).files()[0], self.sen_columns,
self.read_option.output_directory)
print("sen file converted to csv.")
def convert_vhd(self):
VectorConverter.convert_csv(
PathList.match(r"\.vhd$")(
self.read_option.read_directory).files()[0], self.vhd_columns,
self.read_option.output_directory)
print("vhd file converted to csv.")