def main(sat_sensor, in_file):
"""
:param sat_sensor: (str) 卫星对
:param in_file: (str) 输入文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 全局配置接口
out_path = gc.path_mid_ncep
# 程序配置接口
# 卫星配置接口
suffix = sc.ncep2byte_filename_suffix
ncep_table = sc.ncep2byte_ncep_table
######################### 开始处理 ###########################
print "-" * 100
print "Start ncep to byte."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
ymdhm = _get_ymdhm(in_file)
out_file = _get_out_file(in_file, out_path, suffix)
# 判断是否使用新命令进行处理
if int(ymdhm) >= 201501221200:
new = True
else:
new = False
if pb_io.is_none(in_file, out_file, ncep_table):
log.error("Error: {}".format(in_file))
return
ncep2byte = Ncep2Byte(in_file, out_file, new=new, ncep_table=ncep_table)
ncep2byte.ncep2byte()
if not ncep2byte.error:
print ">>> {}".format(ncep2byte.out_file)
else:
log.error("Error: {}".format(in_file))
print "-" * 100
def combine(self):
if self.error:
return
# 如果输出文件已经存在，跳过
elif os.path.isfile(self.ofile):
self.error = True
print "Error: File is already exist, skip it: {}".format(
self.ofile)
return
# 合成日数据
elif pb_io.is_none(self.ifile, self.ofile):
self.error = True
print "Error: Is None: ifile or ofile: {}".format(self.yaml_file)
return
elif len(self.ifile) < 1:
self.error = True
print "Error: File count lower than 1: {}".format(self.yaml_file)
return
for in_file in self.ifile:
if os.path.isfile(in_file):
print "<<< {}".format(in_file)
else:
print "Warning: File is not exist: {}".format(in_file)
continue
self.one_in_file = in_file
# 日合成
with time_block("One combine time:", switch=TIME_TEST):
self._combine_2d()
# 计算数据的平均值
with time_block("Calculate mean time:", switch=TIME_TEST):
print "Start calculate."
self._2d_data_calculate()
# 输出数据集有效数据的数量
self._print_data_count()
def main(sat_sensor, in_file):
"""
绘制 HDF5.dataset 的快视图。支持真彩图和灰度图
:param sat_sensor: 卫星+传感器
:param in_file: HDF5 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
log = LogServer(gc.path_out_log)
# 加载全局配置信息
# 加载程序配置信息
# 加载卫星配置信息
dataset = sc.plt_quick_view_rgb_dataset
rgb_suffix = sc.plt_quick_view_rgb_suffix
colorbar_range = sc.plt_quick_view_img_colorbar_range
log10_set = sc.plt_quick_view_img_log10_set
log10_ticks = sc.plt_quick_view_img_log10_ticks
log10_tick_labels = sc.plt_quick_view_img_log10_tick_labels
# ######################## 开始处理 ###########################
print '-' * 100
print "Start plot quick view picture."
if not os.path.isfile(in_file):
log.error("File not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
in_file_name = os.path.splitext(in_file)[0]
# 绘制真彩图
out_picture = "{}_{}.{}".format(in_file_name, rgb_suffix, "png")
# 如果文件已经存在，跳过
if not os.path.isfile(out_picture):
r_set, g_set, b_set = dataset
rgb = RGB(in_file, r_set, g_set, b_set, out_picture)
rgb.plot()
if not rgb.error:
print ">>> {}".format(out_picture)
print '-' * 100
else:
print "Error: Plot RGB error: {}".format(in_file)
else:
print "File is already exist, skip it: {}".format(out_picture)
# 绘制热度图
for legend in colorbar_range:
dataset_name = legend[0] # 数据集名称
vmax = float(legend[1]) # color bar 范围 最大值
vmin = float(legend[2]) # color bar 范围 最小值
out_picture = "{}_{}.{}".format(in_file_name, dataset_name.replace("Aod", "AOD"), "png")
# 如果文件已经存在，跳过
if os.path.isfile(out_picture):
print "File is already exist, skip it: {}".format(out_picture)
continue
heat_map = {
"vmin": vmin,
"vmax": vmax,
"cmap": "jet",
"fill_value": -32767,
}
if dataset_name in log10_set:
if dataset_name == "Ocean_TSM":
log10_ticks.append(2.00)
log10_tick_labels.append("100")
if dataset_name == "Ocean_YS443":
log10_ticks = log10_ticks[:-3]
log10_tick_labels = log10_tick_labels[:-3]
heat_map["colorbar_ticks"] = log10_ticks
heat_map["colorbar_tick_label"] = log10_tick_labels
lats, lons = get_lats_lons(in_file)
lat_lon_text = _get_lat_lon_text(lats, lons)
if is_none(lats, lons):
lat_lon_line = None
else:
lat_lon_line = {
"lats": lats, # 经度数据集名称
"lons": lons, # 维度数据集名称
"step": 5.0, # 线密度
"line_width": 0.01,
"text": lat_lon_text,
}
quick_view = QuickView(in_file, dataset_name, out_picture, main_view=heat_map,
lat_lon_line=lat_lon_line)
quick_view.plot()
if not quick_view.error:
print ">>> {}".format(quick_view.out_picture)
else:
print "Error: Plot heat view error: {}".format(in_file)
print '-' * 100
def main(pair, ymd):
"""
pair: sat1+sensor1_sat2+sensor2
ymd: str YYYYMMDD
"""
######################### 初始化 ###########################
# 提取参数中的卫星信息和传感器信息
part1, part2 = pair.split("_")
sat1, sensor1 = part1.split("+")
sat2, sensor2 = part2.split("+")
sensor_pair = "{}_{}".format(sensor1, sensor2)
# 判断是静止卫星还是动态卫星
if "FY2" in part1 or "FY4" in part1:
sat_type = "GEOLEO"
elif "FY3" in part1:
sat_type = "LEOLEO"
else:
LOG.error("Cant distinguish the satellite type")
return
# 加载绘图配置文件
plt_cfg_file = os.path.join(MAIN_PATH, "%s_%s_3d.yaml" % (sensor1, sensor2))
plt_cfg = loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
LOG.error("Not find the config file: {}".format(plt_cfg_file))
return
else:
# 加载配置信息
try:
physics_pair = plt_cfg["regression"] # 需要处理的物理量对
if pb_io.is_none(physics_pair):
LOG.error("Yaml args is not completion. : {}".format(plt_cfg_file))
return
except ValueError:
LOG.error("Load yaml config file error, please check it. : {}".format(plt_cfg_file))
return
######################### 开始处理 ###########################
print '-' * 100
print 'Start regression daily.'
# 循环处理每个物理量对
for each in physics_pair:
dict_cabr = {}
dict_cabr_d = {}
dict_cabr_n = {}
dict_bias = {}
dict_bias_d = {}
dict_bias_n = {}
period = plt_cfg[each]["days"] # 回滚天数
# must be in "all", "day", "night"
day_night = ["all", "day", "night"]
if "time" in plt_cfg[each].keys():
day_night = plt_cfg[each]["time"]
for t in day_night:
if t not in ["all", "day", "night"]:
day_night.remove(t)
for idx, chan in enumerate(plt_cfg[each]["chan"]):
LOG.info(u"Start Drawing {} Channel {}".format(each, chan))
oneHDF5 = ReadHDF5()
num_file = period
for daydelta in xrange(period):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
hdf5_name = "COLLOC+%sIR,%s_C_BABJ_%s.hdf5" % (sat_type, pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, hdf5_name)
if not os.path.isfile(filefullpath):
LOG.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if not oneHDF5.LoadData(filefullpath, chan):
LOG.error("Error occur when reading %s of %s" % (chan, filefullpath))
if num_file == 0:
LOG.error(u"No file found.")
continue
elif num_file != period:
LOG.error(u"{} of {} file(s) found.".format(num_file, period))
if is_monthly:
str_time = ymd[:6]
cur_path = os.path.join(MRA_DIR, pair, str_time)
else:
str_time = ymd
cur_path = os.path.join(DRA_DIR, pair, str_time)
# delete 0 in std
if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值，而导致权重极大，所以 std>0 改成 std>0.0001
deletezeros = np.where(oneHDF5.rad1_std > 0.0001)
oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros]
oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len(
oneHDF5.rad1) > 0 else oneHDF5.rad1
oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len(
oneHDF5.rad2) > 0 else oneHDF5.rad2
oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len(
oneHDF5.tbb1) > 0 else oneHDF5.tbb1
oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len(
oneHDF5.tbb2) > 0 else oneHDF5.tbb2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
if len(oneHDF5.ref1_std) > 0.0001:
deletezeros = np.where(oneHDF5.ref1_std > 0.0001)
oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros]
oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len(
oneHDF5.ref1) > 0 else oneHDF5.ref1
oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len(
oneHDF5.ref2) > 0 else oneHDF5.ref2
oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn1) > 0 else oneHDF5.dn1
oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn2) > 0 else oneHDF5.dn2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
# find out day and night
if ("day" in day_night or "night" in day_night) and len(oneHDF5.time) > 0:
vect_is_day = np.vectorize(is_day_timestamp_and_lon)
day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1)
night_index = np.logical_not(day_index)
else:
day_index = None
night_index = None
# 将每个对通用的属性值放到对循环，每个通道用到的属性值放到通道循环
# get threhold, unit, names...
xname, yname = each.split("-")
xname_l = plt_cfg[each]["x_name"]
xunit = plt_cfg[each]["x_unit"]
xlimit = plt_cfg[each]["x_range"][idx]
xmin, xmax = xlimit.split("-")
xmin = float(xmin)
xmax = float(xmax)
yname_l = plt_cfg[each]["y_name"]
yunit = plt_cfg[each]["y_unit"]
ylimit = plt_cfg[each]["y_range"][idx]
ymin, ymax = ylimit.split("-")
ymin = float(ymin)
ymax = float(ymax)
weight = None
if "rad" in xname:
x = oneHDF5.rad1
elif "tbb" in xname:
x = oneHDF5.tbb1
elif "ref" in xname:
x = oneHDF5.ref1
elif "dn" in xname:
x = oneHDF5.dn1
else:
LOG.error("Can't plot %s" % each)
continue
if "rad" in yname:
y = oneHDF5.rad2
elif "tbb" in yname:
y = oneHDF5.tbb2
elif "ref" in yname:
y = oneHDF5.ref2
else:
LOG.error("Can't plot %s" % each)
continue
if "rad" in xname and "rad" in yname:
if len(oneHDF5.rad1_std) > 0:
weight = oneHDF5.rad1_std
o_name = "RadCalCoeff"
elif "tbb" in xname and "tbb" in yname:
o_name = "TBBCalCoeff"
elif "ref" in xname and "ref" in yname:
if len(oneHDF5.ref1_std) > 0:
weight = oneHDF5.ref1_std
o_name = "CorrcCoeff"
elif "dn" in xname and "ref" in yname:
o_name = "CalCoeff"
# 画对角线
if xname == yname:
diagonal = True
else:
diagonal = False
if "all" in day_night and o_name not in dict_cabr:
dict_cabr[o_name] = {}
dict_bias[xname] = {}
if "day" in day_night and o_name not in dict_cabr_d:
dict_cabr_d[o_name] = {}
dict_bias_d[xname] = {}
if "night" in day_night and o_name not in dict_cabr_n:
dict_cabr_n[o_name] = {}
dict_bias_n[xname] = {}
# 对样本点数量进行判断，如果样本点少于 100 个，则不进行绘制
if x.size < 100:
LOG.error("Not enough match point to draw: {}, {}".format(each, chan))
if "all" in day_night:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias[xname][chan] = [np.NaN, np.NaN]
if "day" in day_night:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
if "night" in day_night:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
continue
# regression starts
if "all" in day_night:
o_file = os.path.join(cur_path,
"%s_%s_%s_ALL_%s" % (
pair, o_name, chan, str_time))
print("x_all, y_all", len(x), len(y))
abr, bias = plot(x, y, weight, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr[o_name][chan] = abr
else:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias[xname][chan] = bias
else:
dict_bias[xname][chan] = [np.NaN, np.NaN]
# ------- day ----------
if "day" in day_night:
if day_index is not None and np.where(day_index)[0].size > 10:
o_file = os.path.join(cur_path,
"%s_%s_%s_Day_%s" % (
pair, o_name, chan, str_time))
x_d = x[day_index]
y_d = y[day_index]
w_d = weight[day_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_day, y_day", len(x_d), len(y_d))
abr, bias = plot(x_d, y_d, w_d, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr_d[o_name][chan] = abr
else:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias_d[xname][chan] = bias
else:
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
else:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
# ---------night ------------
if "night" in day_night:
if night_index is not None and np.where(night_index)[0].size > 10:
o_file = os.path.join(cur_path, "%s_%s_%s_Night_%s" % (
pair, o_name, chan, str_time))
x_n = x[night_index]
y_n = y[night_index]
w_n = weight[night_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_night, y_night", len(x_n), len(y_n))
abr, bias = plot(x_n, y_n, w_n, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr_n[o_name][chan] = abr
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias_n[xname][chan] = bias
else:
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
oneHDF5.clear()
# write txt
lock.acquire()
channel = plt_cfg[each]["chan"]
if "all" in day_night:
for o_name in dict_cabr:
write_bias(channel, part1, part2, xname, ymd,
dict_bias, "ALL")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr, "ALL")
if "day" in day_night:
for o_name in dict_cabr_d:
write_bias(channel, part1, part2, xname, ymd,
dict_bias_d, "Day")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr_d, "Day")
if "night" in day_night:
for o_name in dict_cabr_n:
write_bias(channel, part1, part2, xname, ymd,
dict_bias_n, "Night")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr_n, "Night")
lock.release()
def combine(self):
if self.error:
return
# 如果输出文件已经存在，跳过
elif os.path.isfile(self.ofile):
self.error = True
print "File is already exist, skip it: {}".format(self.ofile)
return
# 合成日数据
elif pb_io.is_none(self.ifile, self.pfile, self.ofile):
self.error = True
print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file)
return
elif len(self.ifile) < 1:
self.error = True
print "File count lower than 1: {}".format(self.yaml_file)
fill_value = -32767
for file_idx, in_file in enumerate(self.ifile):
proj_file = self.pfile[file_idx]
if os.path.isfile(in_file) and os.path.isfile(proj_file):
print "*" * 100
print "Start combining file:"
print "<<< {}\n<<< {}".format(in_file, proj_file)
else:
print "File is not exist: {} OR {}".format(in_file, proj_file)
continue
# 加载 proj 数据
self.load_proj_data(proj_file)
# 日合成
with time_block("One combine time:", switch=TIME_TEST):
try:
with h5py.File(in_file, 'r') as h5:
for k in h5.keys():
# 记录属性信息
if k not in self.attrs:
self.attrs[k] = pb_io.attrs2dict(
h5.get(k).attrs)
if k == "Longitude" or k == "Latitude":
continue
elif k not in self.out_data:
if k == "Ocean_Flag":
self.out_data[k] = np.full(
(self.row, self.col),
fill_value,
dtype='i4')
else:
self.out_data[k] = np.full(
(self.row, self.col),
fill_value,
dtype='i2')
# 合并一个数据
proj_data = h5.get(k)[:]
self.out_data[k][self.lut_ii,
self.lut_jj] = proj_data[
self.data_ii, self.data_jj]
except Exception as why:
print why
print "Can't combine file, some error exist: {}".format(
in_file)
with time_block("Grid to lons and lats time:", switch=TIME_TEST):
if "Longitude" not in self.out_data:
lookup_table = prj_core(self.cmd,
self.res,
unit="deg",
row=self.row,
col=self.col)
lookup_table.grid_lonslats()
self.out_data["Longitude"] = lookup_table.lons
self.out_data["Latitude"] = lookup_table.lats
# 输出数据集有效数据的数量
keys = [x for x in self.out_data]
keys.sort()
for k in keys:
if self.out_data[k] is None:
print k
continue
idx = np.where(self.out_data[k] > 0)
print "{:30} : {}".format(k, len(idx[0]))
def combine(self):
all_time_start = datetime.utcnow()
if self.error:
return
# 如果输出文件已经存在，跳过
elif os.path.isfile(self.ofile):
self.error = True
print "File is already exist, skip it: {}".format(self.ofile)
return
# 合成日数据
elif pb_io.is_none(self.ifile, self.pfile, self.ofile):
self.error = True
print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file)
return
elif len(self.ifile) < 1:
self.error = True
print "File count lower than 1: {}".format(self.yaml_file)
self.load_proj_files(self.pfile) # 读取投影文件
fill_value = -32767
with h5py.File(self.ifile[0], 'r') as h5:
data_names = h5.keys()
data_names.sort()
for k in h5.keys():
# 记录属性信息
if k not in self.attrs:
self.attrs[k] = pb_io.attrs2dict(h5.get(k).attrs)
file_count = len(self.ifile)
data_count = list()
date_start = datetime.utcnow()
for data_name in data_names:
if data_name == "Longitude" or data_name == "Latitude" or data_name == "Ocean_Flag":
continue
if 'Azimuth' in data_name or 'Zenith' in data_name or data_name == 'chl':
continue
data = list()
count = 0
# 读取所有的数据
for in_file in self.ifile:
filename = os.path.basename(in_file)
k = filename.split('_1000M')[0]
data_ii, data_jj = self.data_ij[k]
with h5py.File(in_file, 'r') as hdf5:
data_read = hdf5.get(data_name)[:][data_ii, data_jj]
data = np.append(data, data_read)
date_end = datetime.utcnow() - date_start
count += 1
print 'Combine {} {}/{} {}'.format(data_name, count,
file_count, date_end)
condition = np.logical_and(data != 32767, data != -32767)
condition = np.logical_and(condition, data != 0)
lut_ii = self.lut_ii[condition]
lut_jj = self.lut_jj[condition]
data = data[condition]
data_count.append((data_name, len(data)))
if not len(data) == 0:
ijd = dict()
date_start = datetime.utcnow()
for i, j, d in zip(lut_ii, lut_jj, data):
local = (i, j)
if local not in ijd:
ijd[local] = [d]
else:
data_list = ijd[local]
data_list.append(d)
date_end = datetime.utcnow() - date_start
print date_end
local = ijd.keys()
local = np.array(local)
lut_ii = local[:, 0].astype(np.int16)
lut_jj = local[:, 1].astype(np.int16)
data = ijd.values()
date_start = datetime.utcnow()
mean_ = np.vectorize(mean_data)
data = mean_(data)
date_end = datetime.utcnow() - date_start
print date_end
out_data = np.full((self.row, self.col),
fill_value,
dtype='i2')
data = data.reshape(-1)
out_data[lut_ii, lut_jj] = data
else:
out_data = np.full((self.row, self.col),
fill_value,
dtype='i2')
self.write(out_data, data_name, self.ofile)
# 输出经纬度数据集
lookup_table = prj_core(self.cmd,
self.res,
unit="deg",
row=self.row,
col=self.col)
lookup_table.grid_lonslats()
longitude = lookup_table.lons
latitude = lookup_table.lats
data_name = "Longitude"
self.write(longitude, data_name, self.ofile)
data_name = "Latitude"
self.write(latitude, data_name, self.ofile)
# 输出海陆标记和方位角
for data_name in [
'SensorAzimuth', 'SensorZenith', 'SolarAzimuth', 'SolarZenith',
'Ocean_Flag', 'chl'
]:
out_data = np.full((self.row, self.col), fill_value, dtype='i4')
data = list()
for in_file in self.ifile:
filename = os.path.basename(in_file)
k = filename.split('_1000M')[0]
data_ii, data_jj = self.data_ij[k]
with h5py.File(in_file, 'r') as hdf5:
data_read = hdf5.get(data_name)[:][data_ii, data_jj]
data = np.append(data, data_read)
data = np.reshape(data, -1)
lut_ii = self.lut_ii.astype(np.int16)
lut_jj = self.lut_jj.astype(np.int16)
out_data[lut_ii, lut_jj] = data
self.write(out_data, data_name, self.ofile)
# 输出数据集有效数据的数量
for count in data_count:
print "{:30} : {}".format(count[0], count[1])
all_time_end = datetime.utcnow() - all_time_start
print all_time_end