def __init__(self, fmt):
self.format_key = format_key()
self.format_type = format_type()
self.file_format = file_format()
self.index_dict = index_dict()
self.output_format = output_format()
self.multi_line_header = multi_line_header()
self.useless_columns = useless_column()
self.fmt = fmt
self.data = {}
self.stations = None
def pack_data(path,
file_name,
fmt='on',
types=None,
combine_model=True,
model_columns=None,
restore_columns=None,
ori_order=True):
format_keys = format_key()
if not types:
types = format_keys['types']
if not restore_columns:
restore_columns = restore_column()
types = [t for t in types if t in restore_columns]
all_columns = get_all_column(types, file_format()[fmt])
name_indices = name_index_dict()
if combine_model:
if not model_columns:
model_columns = model_column()
models = {}
for t in types:
miss = [col for col in restore_columns[t] if col not in all_columns[t]]
restore_columns[t] = [
col for col in restore_columns[t]
if col in all_columns[t] and col not in name_indices[t]
]
restore_columns[t] = restore_columns[t] + ['flag']
if combine_model:
miss.extend(
[col for col in model_columns[t] if col not in all_columns[t]])
model_columns[t] = [
col for col in model_columns[t]
if col in all_columns[t] and col not in name_indices[t]
]
if miss:
print('[%s] data miss: ' % t, miss)
data = dict([(t, {}) for t in types])
for d in os.listdir(path):
if not os.path.exists(os.path.join(path, d, 'LF.L1')):
continue
power = Power(fmt=fmt)
power.load_power(os.path.join(path, d), fmt=fmt, station=False)
for t in types:
power.data[t].set_index(name_indices[t], inplace=True)
if ori_order and 'ori_order' in power.data[t]:
data[t][d] = power.data[t][restore_columns[t] + ['ori_order']]
else:
data[t][d] = power.data[t][restore_columns[t]]
if combine_model:
if t in models:
idx = power.data[t].index.difference(models[t].index)
models[t] = models[t].append(
power.data[t].loc[idx, model_columns[t]])
else:
models[t] = power.data[t][model_columns[t]]
package = {}
for t in types:
package[t] = pd.concat(data[t].values(), keys=data[t].keys())
if combine_model and t in models:
package['model_' + t] = models[t]
hdf = pd.HDFStore(file_name, 'w', complevel=9, complib='blosc')
for k in package:
hdf.put(key=k, value=package[k])
hdf.close()