def read_file_attr(in_file):
try:
with h5py.File(in_file, 'r') as hdf5_file:
return attrs2dict(hdf5_file.attrs)
except Exception as why:
print why
return
def read_file_attr(in_file):
try:
hdf4 = SD(in_file, SDC.READ)
return attrs2dict(hdf4.attributes())
except Exception as why:
print 'ReadHDF4.read_file_attr: {}'.format(why)
return
def _combine_2d(self):
"""
将日数据合成为2维数据，然后计算均值
"""
if self.error:
return
try:
with h5py.File(self.one_in_file, 'r') as h5:
for k in h5.keys():
if k not in self.out_data: # 创建输出数据集和计数器
shape = h5.get(k).shape
if k == "Ocean_Flag":
continue
elif k == "Longitude" or k == "Latitude":
self.out_data[k] = h5.get(k)[:]
else:
self.out_data[k] = np.zeros(shape, dtype='i2')
self.counter[k] = np.zeros(shape, dtype='i2')
if k == "Longitude" or k == "Latitude" or k == "Ocean_Flag":
continue
else:
value = h5.get(k)[:]
idx = np.where(value > 0)
self.out_data[k][
idx] = self.out_data[k][idx] + value[idx]
self.counter[k][idx] += 1
# 记录属性信息
if k not in self.attrs:
self.attrs[k] = pb_io.attrs2dict(h5.get(k).attrs)
except Exception as why:
print why
print "Can't combine file, some error exist: {}".format(
self.one_in_file)
def set_file_attr(self):
"""
get hdf5 file attrs self.file_attr
:return:
"""
if self.resolution == 4000:
satellite_type1 = ['FY4A']
if self.satellite in satellite_type1:
with h5py.File(self.in_file, 'r') as h5r:
self.file_attr = attrs2dict(h5r.attrs)
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def set_file_attr(self):
"""
get hdf5 file attrs self.file_attr
:return:
"""
if self.resolution == 13500:
satellite_type = ['AQUA', 'TERRA']
if self.satellite in satellite_type:
h4r = SD(self.in_file, SDC.READ)
self.file_attr = attrs2dict(h4r.attributes())
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def set_file_attr(self):
"""
根据 self.file_level_1 获取 L1 文件的属性
set self.level_1_attr
储存格式是字典
:return:
"""
if self.resolution == 17000:
satellite_type1 = ['FY3A', 'FY3B', 'FY3C']
if self.satellite in satellite_type1:
with h5py.File(self.in_file, 'r') as hdf5_file:
self.file_attr = attrs2dict(hdf5_file.attrs)
else:
raise ValueError(
'Cant read this satellite`s data.: {}'.format(self.satellite))
else:
raise ValueError(
"Cant handle this resolution: ".format(self.resolution))
def set_file_attr(self):
"""
根据 self.file_level_1 获取 L1 文件的属性
set self.level_1_attr
储存格式是字典
:return:
"""
if self.resolution == 1000:
satellite_type1 = ['FY1C', 'FY1D']
if self.satellite in satellite_type1:
hdf4 = SD(self.in_file, SDC.READ)
self.file_attr = attrs2dict(hdf4.attributes())
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def set_file_attr(self):
"""
get hdf5 file attrs self.file_attr
:return:
"""
if self.resolution == 5000:
satellite_type1 = ['FY2C', 'FY2D', 'FY2E', 'FY2F', 'FY2G', 'FY2H']
if self.satellite in satellite_type1:
with h5py.File(self.in_file, 'r') as h5r:
NOMCenterLon = h5r.get('NomFileInfo')['NOMCenterLon'][0]
a = {}
a['NOMCenterLon'] = NOMCenterLon
self.file_attr = attrs2dict(a)
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def _combine_3d(self):
"""
将二维数据合成三维数据
"""
if self.error:
return
try:
with h5py.File(self.one_in_file, 'r') as h5:
for k in h5.keys():
shape = h5.get(k).shape
reshape = (shape[0], shape[1], 1)
if k not in self.out_data: # 创建输出数据集和计数器
if k == "Ocean_Flag":
continue
elif k == "Longitude" or k == "Latitude":
self.out_data[k] = h5.get(k)[:]
else:
data = h5.get(k)[:].reshape(reshape)
self.out_data[k] = data
else:
if k == "Longitude" or k == "Latitude" or k == "Ocean_Flag":
continue
else:
data = h5.get(k)[:].reshape(reshape)
self.out_data[k] = np.concatenate(
(self.out_data[k], data), axis=2)
# 记录属性信息
if k in self.attrs:
continue
else:
self.attrs[k] = pb_io.attrs2dict(h5.get(k).attrs)
except Exception as why:
print why
print "Can't combine file, some error exist: {}".format(
self.one_in_file)
def combine(self):
if self.error:
return
# 如果输出文件已经存在，跳过
elif os.path.isfile(self.ofile):
self.error = True
print "File is already exist, skip it: {}".format(self.ofile)
return
# 合成日数据
elif pb_io.is_none(self.ifile, self.pfile, self.ofile):
self.error = True
print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file)
return
elif len(self.ifile) < 1:
self.error = True
print "File count lower than 1: {}".format(self.yaml_file)
fill_value = -32767
for file_idx, in_file in enumerate(self.ifile):
proj_file = self.pfile[file_idx]
if os.path.isfile(in_file) and os.path.isfile(proj_file):
print "*" * 100
print "Start combining file:"
print "<<< {}\n<<< {}".format(in_file, proj_file)
else:
print "File is not exist: {} OR {}".format(in_file, proj_file)
continue
# 加载 proj 数据
self.load_proj_data(proj_file)
# 日合成
with time_block("One combine time:", switch=TIME_TEST):
try:
with h5py.File(in_file, 'r') as h5:
for k in h5.keys():
# 记录属性信息
if k not in self.attrs:
self.attrs[k] = pb_io.attrs2dict(
h5.get(k).attrs)
if k == "Longitude" or k == "Latitude":
continue
elif k not in self.out_data:
if k == "Ocean_Flag":
self.out_data[k] = np.full(
(self.row, self.col),
fill_value,
dtype='i4')
else:
self.out_data[k] = np.full(
(self.row, self.col),
fill_value,
dtype='i2')
# 合并一个数据
proj_data = h5.get(k)[:]
self.out_data[k][self.lut_ii,
self.lut_jj] = proj_data[
self.data_ii, self.data_jj]
except Exception as why:
print why
print "Can't combine file, some error exist: {}".format(
in_file)
with time_block("Grid to lons and lats time:", switch=TIME_TEST):
if "Longitude" not in self.out_data:
lookup_table = prj_core(self.cmd,
self.res,
unit="deg",
row=self.row,
col=self.col)
lookup_table.grid_lonslats()
self.out_data["Longitude"] = lookup_table.lons
self.out_data["Latitude"] = lookup_table.lats
# 输出数据集有效数据的数量
keys = [x for x in self.out_data]
keys.sort()
for k in keys:
if self.out_data[k] is None:
print k
continue
idx = np.where(self.out_data[k] > 0)
print "{:30} : {}".format(k, len(idx[0]))
def combine(self):
all_time_start = datetime.utcnow()
if self.error:
return
# 如果输出文件已经存在，跳过
elif os.path.isfile(self.ofile):
self.error = True
print "File is already exist, skip it: {}".format(self.ofile)
return
# 合成日数据
elif pb_io.is_none(self.ifile, self.pfile, self.ofile):
self.error = True
print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file)
return
elif len(self.ifile) < 1:
self.error = True
print "File count lower than 1: {}".format(self.yaml_file)
self.load_proj_files(self.pfile) # 读取投影文件
fill_value = -32767
with h5py.File(self.ifile[0], 'r') as h5:
data_names = h5.keys()
data_names.sort()
for k in h5.keys():
# 记录属性信息
if k not in self.attrs:
self.attrs[k] = pb_io.attrs2dict(h5.get(k).attrs)
file_count = len(self.ifile)
data_count = list()
date_start = datetime.utcnow()
for data_name in data_names:
if data_name == "Longitude" or data_name == "Latitude" or data_name == "Ocean_Flag":
continue
if 'Azimuth' in data_name or 'Zenith' in data_name or data_name == 'chl':
continue
data = list()
count = 0
# 读取所有的数据
for in_file in self.ifile:
filename = os.path.basename(in_file)
k = filename.split('_1000M')[0]
data_ii, data_jj = self.data_ij[k]
with h5py.File(in_file, 'r') as hdf5:
data_read = hdf5.get(data_name)[:][data_ii, data_jj]
data = np.append(data, data_read)
date_end = datetime.utcnow() - date_start
count += 1
print 'Combine {} {}/{} {}'.format(data_name, count,
file_count, date_end)
condition = np.logical_and(data != 32767, data != -32767)
condition = np.logical_and(condition, data != 0)
lut_ii = self.lut_ii[condition]
lut_jj = self.lut_jj[condition]
data = data[condition]
data_count.append((data_name, len(data)))
if not len(data) == 0:
ijd = dict()
date_start = datetime.utcnow()
for i, j, d in zip(lut_ii, lut_jj, data):
local = (i, j)
if local not in ijd:
ijd[local] = [d]
else:
data_list = ijd[local]
data_list.append(d)
date_end = datetime.utcnow() - date_start
print date_end
local = ijd.keys()
local = np.array(local)
lut_ii = local[:, 0].astype(np.int16)
lut_jj = local[:, 1].astype(np.int16)
data = ijd.values()
date_start = datetime.utcnow()
mean_ = np.vectorize(mean_data)
data = mean_(data)
date_end = datetime.utcnow() - date_start
print date_end
out_data = np.full((self.row, self.col),
fill_value,
dtype='i2')
data = data.reshape(-1)
out_data[lut_ii, lut_jj] = data
else:
out_data = np.full((self.row, self.col),
fill_value,
dtype='i2')
self.write(out_data, data_name, self.ofile)
# 输出经纬度数据集
lookup_table = prj_core(self.cmd,
self.res,
unit="deg",
row=self.row,
col=self.col)
lookup_table.grid_lonslats()
longitude = lookup_table.lons
latitude = lookup_table.lats
data_name = "Longitude"
self.write(longitude, data_name, self.ofile)
data_name = "Latitude"
self.write(latitude, data_name, self.ofile)
# 输出海陆标记和方位角
for data_name in [
'SensorAzimuth', 'SensorZenith', 'SolarAzimuth', 'SolarZenith',
'Ocean_Flag', 'chl'
]:
out_data = np.full((self.row, self.col), fill_value, dtype='i4')
data = list()
for in_file in self.ifile:
filename = os.path.basename(in_file)
k = filename.split('_1000M')[0]
data_ii, data_jj = self.data_ij[k]
with h5py.File(in_file, 'r') as hdf5:
data_read = hdf5.get(data_name)[:][data_ii, data_jj]
data = np.append(data, data_read)
data = np.reshape(data, -1)
lut_ii = self.lut_ii.astype(np.int16)
lut_jj = self.lut_jj.astype(np.int16)
out_data[lut_ii, lut_jj] = data
self.write(out_data, data_name, self.ofile)
# 输出数据集有效数据的数量
for count in data_count:
print "{:30} : {}".format(count[0], count[1])
all_time_end = datetime.utcnow() - all_time_start
print all_time_end
