def __init__(self, fmt): self.format_key = format_key() self.format_type = format_type() self.file_format = file_format() self.index_dict = index_dict() self.output_format = output_format() self.multi_line_header = multi_line_header() self.useless_columns = useless_column() self.fmt = fmt self.data = {} self.stations = None
def pack_data(path, file_name, fmt='on', types=None, combine_model=True, model_columns=None, restore_columns=None, ori_order=True): format_keys = format_key() if not types: types = format_keys['types'] if not restore_columns: restore_columns = restore_column() types = [t for t in types if t in restore_columns] all_columns = get_all_column(types, file_format()[fmt]) name_indices = name_index_dict() if combine_model: if not model_columns: model_columns = model_column() models = {} for t in types: miss = [col for col in restore_columns[t] if col not in all_columns[t]] restore_columns[t] = [ col for col in restore_columns[t] if col in all_columns[t] and col not in name_indices[t] ] restore_columns[t] = restore_columns[t] + ['flag'] if combine_model: miss.extend( [col for col in model_columns[t] if col not in all_columns[t]]) model_columns[t] = [ col for col in model_columns[t] if col in all_columns[t] and col not in name_indices[t] ] if miss: print('[%s] data miss: ' % t, miss) data = dict([(t, {}) for t in types]) for d in os.listdir(path): if not os.path.exists(os.path.join(path, d, 'LF.L1')): continue power = Power(fmt=fmt) power.load_power(os.path.join(path, d), fmt=fmt, station=False) for t in types: power.data[t].set_index(name_indices[t], inplace=True) if ori_order and 'ori_order' in power.data[t]: data[t][d] = power.data[t][restore_columns[t] + ['ori_order']] else: data[t][d] = power.data[t][restore_columns[t]] if combine_model: if t in models: idx = power.data[t].index.difference(models[t].index) models[t] = models[t].append( power.data[t].loc[idx, model_columns[t]]) else: models[t] = power.data[t][model_columns[t]] package = {} for t in types: package[t] = pd.concat(data[t].values(), keys=data[t].keys()) if combine_model and t in models: package['model_' + t] = models[t] hdf = pd.HDFStore(file_name, 'w', complevel=9, complib='blosc') for k in package: hdf.put(key=k, value=package[k]) hdf.close()