def run(pair, ymd):
"""
pair: sat1+sensor1_sat2+sensor2
ymd: str YYYYMMDD
"""
# 提取参数中的卫星信息和传感器信息
part1, part2 = pair.split("_")
sat1, sensor1 = part1.split("+")
sat2, sensor2 = part2.split("+")
# 判断是静止卫星还是动态卫星
if "FY2" in part1 or "FY4" in part1:
Type = "GEOLEO"
elif "FY3" in part1:
Type = "LEOLEO"
else:
LOG.error("Cant distinguish the satellite type")
return
# 加载绘图配置文件
plt_cfg_file = os.path.join(MAIN_PATH, "cfg", "%s.plt" % pair)
plt_cfg = loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
LOG.error("Not find the config file: {}".format(plt_cfg_file))
return
PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数
ym = ymd[:6]
ymd = ym + '%02d' % PERIOD # 当月最后一天
LOG.info(u"----- Start Drawing Monthly TBBias Analysis Pic, PAIR: {}, YMD: {} -----".format(pair, ymd))
for each in plt_cfg['monthly_staistics']:
# Day_Night must be in 'all', 'day', 'night'
Day_Night = ['all', 'day', 'night'] # default
if 'time' in plt_cfg[each].keys():
Day_Night = plt_cfg[each]['time']
for i in Day_Night:
if i not in ['all', 'day', 'night']:
Day_Night.remove(i)
for idx, chan in enumerate(plt_cfg[each]['chan']):
LOG.info(u"Start Drawing {} Channel {}".format(each, chan))
oneHDF5 = ReadHDF5()
# load Matched HDF5
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.hdf5' % (Type, pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, nc_name)
if not os.path.isfile(filefullpath):
LOG.info(u"HDF5 not found: {}".format(filefullpath))
num_file -= 1
continue
if not oneHDF5.LoadData(filefullpath, chan):
LOG.error('Error occur when reading %s of %s' % (chan, filefullpath))
if num_file == 0:
LOG.error(u"No file found.")
continue
elif num_file != PERIOD:
LOG.info(u"{} of {} file(s) found.".format(num_file, PERIOD))
# 输出目录
cur_path = os.path.join(MBA_DIR, pair, ymd[:6])
# get threhold, unit, names...
xname, yname = each.split('-')
bias = xname
xname_l = xname.upper()
xunit = plt_cfg[each]['x_unit']
xlimit = plt_cfg[each]['x_range'][idx]
xmin, xmax = xlimit.split('-')
xmin = float(xmin)
xmax = float(xmax)
# delete 0 in std
if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值，而导致权重极大，所以 std>0 改成 std>0.0001
deletezeros = np.where(oneHDF5.rad1_std > 0.0001)
oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros]
oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len(
oneHDF5.rad1) > 0 else oneHDF5.rad1
oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len(
oneHDF5.rad2) > 0 else oneHDF5.rad2
oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len(
oneHDF5.tbb1) > 0 else oneHDF5.tbb1
oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len(
oneHDF5.tbb2) > 0 else oneHDF5.tbb2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
if len(oneHDF5.ref1_std) > 0.0001:
deletezeros = np.where(oneHDF5.ref1_std > 0.0001)
oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros]
oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len(
oneHDF5.ref1) > 0 else oneHDF5.ref1
oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len(
oneHDF5.ref2) > 0 else oneHDF5.ref2
oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn1) > 0 else oneHDF5.dn1
oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn2) > 0 else oneHDF5.dn2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
# find out day and night
if ('day' in Day_Night or 'night' in Day_Night) and len(oneHDF5.time) > 0:
vect_is_day = np.vectorize(is_day_timestamp_and_lon)
day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1)
night_index = np.logical_not(day_index)
else:
day_index = None
night_index = None
# get x
dset_name = xname + "1"
if hasattr(oneHDF5, dset_name):
x = getattr(oneHDF5, dset_name)
else:
LOG.error("Can't plot, no %s in HDF5 class" % dset_name)
continue
# get y
dset_name = yname + "2"
if hasattr(oneHDF5, dset_name):
y = getattr(oneHDF5, dset_name)
else:
LOG.error("Can't plot, no %s in HDF5 class" % dset_name)
continue
if 'rad' == bias:
o_name = 'RadBiasMonthStats'
elif 'tbb' == bias:
o_name = 'TBBiasMonthStats'
elif 'ref' == bias:
o_name = 'RefBiasMonthStats'
else:
o_name = 'DUMMY'
if x.size < 10:
LOG.error("Not enough match point to draw.")
continue
# 获取 std
weight = None
if 'rad' in xname and 'rad' in yname:
if len(oneHDF5.rad1_std) > 0:
weight = oneHDF5.rad1_std
elif 'tbb' in xname and 'tbb' in yname:
weight = None
elif 'ref' in xname and 'ref' in yname:
if len(oneHDF5.ref1_std) > 0:
weight = oneHDF5.ref1_std
elif 'dn' in xname and 'ref' in yname:
weight = None
# rad-specified regression starts
reference_list = []
if 'reference' in plt_cfg[each]:
reference_list = plt_cfg[each]['reference'][idx]
if 'all' in Day_Night:
o_file = os.path.join(cur_path,
'%s_%s_%s_ALL_%s' % (pair, o_name, chan, ym))
print("x_all, y_all", len(x), len(y))
plot(x, y, weight, o_file,
part1, part2, chan, ym, 'ALL', reference_list,
xname, xname_l, xunit, xmin, xmax)
# ------- day ----------
if 'day' in Day_Night:
if day_index is not None and np.where(day_index)[0].size > 10:
# rad-specified
o_file = os.path.join(cur_path,
'%s_%s_%s_Day_%s' % (pair, o_name, chan, ym))
x_d = x[day_index]
y_d = y[day_index]
w_d = weight[day_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_day, y_day", len(x_d), len(y_d))
plot(x_d, y_d, w_d, o_file,
part1, part2, chan, ym, 'Day', reference_list,
xname, xname_l, xunit, xmin, xmax)
if 'night' in Day_Night:
# ---------night ------------
if night_index is not None and np.where(night_index)[0].size > 10:
# rad-specified
o_file = os.path.join(cur_path,
'%s_%s_%s_Night_%s' % (pair, o_name, chan, ym))
x_n = x[night_index]
y_n = y[night_index]
w_n = weight[day_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_night, y_night", len(x_n), len(y_n))
plot(x_n, y_n, w_n, o_file,
part1, part2, chan, ym, 'Night', reference_list,
xname, xname_l, xunit, xmin, xmax)
def run(pair, ymd, is_monthly):
"""
pair: sat1+sensor1_sat2+sensor2
ymd: str YYYYMMDD
"""
# 提取参数中的卫星信息和传感器信息
part1, part2 = pair.split("_")
sat1, sensor1 = part1.split("+")
sat2, sensor2 = part2.split("+")
# 判断是静止卫星还是动态卫星
if "FY2" in part1 or "FY4" in part1:
Type = "GEOLEO"
elif "FY3" in part1:
Type = "LEOLEO"
else:
Log.error("Cant distinguish the satellite type")
return
# 加载绘图配置文件
plt_cfg_file = os.path.join(MainPath, "%s_%s_3d.yaml" % (sensor1, sensor2))
plt_cfg = loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
Log.error("Not find the config file: {}".format(plt_cfg_file))
return
Log.info(u"----- Start Drawing Regression-Pic, PAIR: {}, YMD: {} -----".format(pair, ymd))
for each in plt_cfg["regression"]:
dict_cabr = {}
dict_cabr_d = {}
dict_cabr_n = {}
dict_bias = {}
dict_bias_d = {}
dict_bias_n = {}
# 需要回滚的天数
if is_monthly:
PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数
ymd = ymd[:6] + "%02d" % PERIOD # 当月最后一天
else:
PERIOD = plt_cfg[each]["days"]
# must be in "all", "day", "night"
Day_Night = ["all", "day", "night"]
if "time" in plt_cfg[each].keys():
Day_Night = plt_cfg[each]["time"]
for t in Day_Night:
if t not in ["all", "day", "night"]:
Day_Night.remove(t)
for idx, chan in enumerate(plt_cfg[each]["chan"]):
Log.info(u"Start Drawing {} Channel {}".format(each, chan))
oneHDF5 = ReadHDF5()
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
hdf5_name = "COLLOC+%sIR,%s_C_BABJ_%s.hdf5" % (Type, pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, hdf5_name)
if not os.path.isfile(filefullpath):
Log.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if not oneHDF5.LoadData(filefullpath, chan):
Log.error("Error occur when reading %s of %s" % (chan, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
continue
elif num_file != PERIOD:
Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD))
if is_monthly:
str_time = ymd[:6]
cur_path = os.path.join(MRA_DIR, pair, str_time)
else:
str_time = ymd
cur_path = os.path.join(DRA_DIR, pair, str_time)
# delete 0 in std
if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值，而导致权重极大，所以 std>0 改成 std>0.0001
deletezeros = np.where(oneHDF5.rad1_std > 0.0001)
oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros]
oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len(
oneHDF5.rad1) > 0 else oneHDF5.rad1
oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len(
oneHDF5.rad2) > 0 else oneHDF5.rad2
oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len(
oneHDF5.tbb1) > 0 else oneHDF5.tbb1
oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len(
oneHDF5.tbb2) > 0 else oneHDF5.tbb2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
if len(oneHDF5.ref1_std) > 0.0001:
deletezeros = np.where(oneHDF5.ref1_std > 0.0001)
oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros]
oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len(
oneHDF5.ref1) > 0 else oneHDF5.ref1
oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len(
oneHDF5.ref2) > 0 else oneHDF5.ref2
oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn1) > 0 else oneHDF5.dn1
oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len(
oneHDF5.dn2) > 0 else oneHDF5.dn2
oneHDF5.time = oneHDF5.time[deletezeros] if len(
oneHDF5.time) > 0 else oneHDF5.time
oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len(
oneHDF5.lon1) > 0 else oneHDF5.lon1
oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len(
oneHDF5.lon2) > 0 else oneHDF5.lon2
# find out day and night
if ("day" in Day_Night or "night" in Day_Night) and len(oneHDF5.time) > 0:
vect_is_day = np.vectorize(is_day_timestamp_and_lon)
day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1)
night_index = np.logical_not(day_index)
else:
day_index = None
night_index = None
# 将每个对通用的属性值放到对循环，每个通道用到的属性值放到通道循环
# get threhold, unit, names...
xname, yname = each.split("-")
xname_l = plt_cfg[each]["x_name"]
xunit = plt_cfg[each]["x_unit"]
xlimit = plt_cfg[each]["x_range"][idx]
xmin, xmax = xlimit.split("-")
xmin = float(xmin)
xmax = float(xmax)
yname_l = plt_cfg[each]["y_name"]
yunit = plt_cfg[each]["y_unit"]
ylimit = plt_cfg[each]["y_range"][idx]
ymin, ymax = ylimit.split("-")
ymin = float(ymin)
ymax = float(ymax)
weight = None
if "rad" in xname:
x = oneHDF5.rad1
elif "tbb" in xname:
x = oneHDF5.tbb1
elif "ref" in xname:
x = oneHDF5.ref1
elif "dn" in xname:
x = oneHDF5.dn1
else:
Log.error("Can't plot %s" % each)
continue
if "rad" in yname:
y = oneHDF5.rad2
elif "tbb" in yname:
y = oneHDF5.tbb2
elif "ref" in yname:
y = oneHDF5.ref2
else:
Log.error("Can't plot %s" % each)
continue
if "rad" in xname and "rad" in yname:
if len(oneHDF5.rad1_std) > 0:
weight = oneHDF5.rad1_std
o_name = "RadCalCoeff"
elif "tbb" in xname and "tbb" in yname:
o_name = "TBBCalCoeff"
elif "ref" in xname and "ref" in yname:
if len(oneHDF5.ref1_std) > 0:
weight = oneHDF5.ref1_std
o_name = "CorrcCoeff"
elif "dn" in xname and "ref" in yname:
o_name = "CalCoeff"
# 画对角线
if xname == yname:
diagonal = True
else:
diagonal = False
if "all" in Day_Night and o_name not in dict_cabr:
dict_cabr[o_name] = {}
dict_bias[xname] = {}
if "day" in Day_Night and o_name not in dict_cabr_d:
dict_cabr_d[o_name] = {}
dict_bias_d[xname] = {}
if "night" in Day_Night and o_name not in dict_cabr_n:
dict_cabr_n[o_name] = {}
dict_bias_n[xname] = {}
# 对样本点数量进行判断，如果样本点少于 100 个，则不进行绘制
if x.size < 100:
Log.error("Not enough match point to draw: {}, {}".format(each, chan))
if "all" in Day_Night:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias[xname][chan] = [np.NaN, np.NaN]
if "day" in Day_Night:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
if "night" in Day_Night:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
continue
# regression starts
if "all" in Day_Night:
o_file = os.path.join(cur_path,
"%s_%s_%s_ALL_%s" % (
pair, o_name, chan, str_time))
print("x_all, y_all", len(x), len(y))
abr, bias = plot(x, y, weight, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr[o_name][chan] = abr
else:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias[xname][chan] = bias
else:
dict_bias[xname][chan] = [np.NaN, np.NaN]
# ------- day ----------
if "day" in Day_Night:
if day_index is not None and np.where(day_index)[0].size > 10:
o_file = os.path.join(cur_path,
"%s_%s_%s_Day_%s" % (
pair, o_name, chan, str_time))
x_d = x[day_index]
y_d = y[day_index]
w_d = weight[day_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_day, y_day", len(x_d), len(y_d))
abr, bias = plot(x_d, y_d, w_d, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr_d[o_name][chan] = abr
else:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias_d[xname][chan] = bias
else:
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
else:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_d[xname][chan] = [np.NaN, np.NaN]
# ---------night ------------
if "night" in Day_Night:
if night_index is not None and np.where(night_index)[0].size > 10:
o_file = os.path.join(cur_path, "%s_%s_%s_Night_%s" % (
pair, o_name, chan, str_time))
x_n = x[night_index]
y_n = y[night_index]
w_n = weight[night_index] if weight is not None else None
print("x_all, y_all", len(x), len(y))
print("x_night, y_night", len(x_n), len(y_n))
abr, bias = plot(x_n, y_n, w_n, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax,
diagonal, is_monthly)
if abr:
dict_cabr_n[o_name][chan] = abr
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if bias:
dict_bias_n[xname][chan] = bias
else:
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
dict_bias_n[xname][chan] = [np.NaN, np.NaN]
oneHDF5.clear()
# write txt
lock.acquire()
channel = plt_cfg[each]["chan"]
if "all" in Day_Night:
for o_name in dict_cabr:
write_bias(channel, part1, part2, xname, ymd,
dict_bias, "ALL")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr, "ALL")
if "day" in Day_Night:
for o_name in dict_cabr_d:
write_bias(channel, part1, part2, xname, ymd,
dict_bias_d, "Day")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr_d, "Day")
if "night" in Day_Night:
for o_name in dict_cabr_n:
write_bias(channel, part1, part2, xname, ymd,
dict_bias_n, "Night")
write_cabr(channel, part1, part2, o_name, ymd,
dict_cabr_n, "Night")
lock.release()
def proj_dcc(self):
# 初始化投影参数 rowMax=None, colMax=None
lookup_table = prj_gll(resLat=self.resLat, resLon=self.resLon)
newLats, newLons = lookup_table.generateLatsLons()
proj_data_num = np.full_like(newLats, 0)
row = proj_data_num.shape[0]
col = proj_data_num.shape[1]
if self.is_monthly: # 如果是月的，需要查找当前自然月所有数据
PERIOD = calendar.monthrange(int(self.ymd[:4]),
int(self.ymd[4:6]))[1] # 当前月份天数
_ymd = self.ymd[:6] + '%02d' % PERIOD # 当月最后一天
else:
PERIOD = 1
_ymd = self.ymd
file_list = []
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(_ymd, -daydelta)
# 判断文件是否存在
file_name = '%s_DCC_SLT_%s.H5' % (self.sat_sensor, cur_ymd)
ym = self.ymd[:6]
HDF_file = os.path.join(self.ifile, ym, file_name)
if not os.path.isfile(HDF_file):
print("file no exist : %s" % HDF_file)
else:
file_list.append(HDF_file)
if len(file_list) == 0:
return
# 开始进行投影
for HDF_file in file_list:
h5File = h5py.File(HDF_file, 'r')
lons = h5File.get('Longitude')[:]
lats = h5File.get('Latitude')[:]
h5File.close()
lons = lons / 100.
lats = lats / 100.
ii, jj = lookup_table.lonslats2ij(lons, lats)
for i in xrange(row):
for j in xrange(col):
condition = np.logical_and(ii[:] == i, jj[:] == j)
idx = np.where(condition)
proj_data_num[i][j] = proj_data_num[i][j] + len(idx[0])
proj_data_num = np.ma.masked_where(proj_data_num == 0, proj_data_num)
if self.is_monthly:
print("plot: %s" % self.ymd[0:6])
p = dv_map.dv_map()
p.easyplot(newLats,
newLons,
proj_data_num,
vmin=0,
vmax=10000,
ptype=None,
markersize=20,
marker='s')
title_name = '%s_dcc_projection_' % self.sat_sensor + str(
self.ymd[0:6])
p.title = u'dcc： ' + str(title_name) + u' (分辨率1度)'
opath_fig = os.path.join(self.ofile, 'Monthly',
'%s' % self.ymd[:6])
if not os.path.exists(opath_fig):
os.makedirs(opath_fig)
fig_name = os.path.join(
opath_fig,
'%s_%s_dcc_monthly.png' % (self.sat_sensor, self.ymd[:6]))
p.savefig(fig_name)
opath_hdf = os.path.join(self.ofile, 'Monthly',
'%s' % self.ymd[:6])
if not os.path.exists(opath_hdf):
os.mkdir(opath_hdf)
opath_hdf = os.path.join(
opath_hdf,
'%s_%s_dcc_monthly.hdf' % (self.sat_sensor, self.ymd[0:6]))
h5file_W = h5py.File(opath_hdf, 'w')
h5file_W.create_dataset('proj_data_nums',
dtype='i2',
data=proj_data_num,
compression='gzip',
compression_opts=5,
shuffle=True)
h5file_W.close()
###########################################
# read hdf file and cont values
print("write txt: %s" % self.ymd[0:6])
if not os.path.exists(self.ofile_txt):
os.mkdir(self.ofile_txt)
opath_txt = os.path.join(
self.ofile_txt, '%s_dcc_monthly_count.txt' % self.sat_sensor)
if self.ifile and len(self.ymd) == 8:
hdf = h5py.File(opath_hdf, 'r')
data_mat = hdf.get("proj_data_nums")[:]
hdf.close()
count_value = np.sum(data_mat)
self.data_count = '%8s\t%8s\n' % (self.ymd[0:6], count_value)
lock.acquire()
self.write_txt(opath_txt)
lock.release()
else:
print("plot: %s" % self.ymd)
p = dv_map.dv_map()
p.easyplot(newLats,
newLons,
proj_data_num,
vmin=0,
vmax=10000,
ptype=None,
markersize=20,
marker='s')
title_name = '%s_dcc_projection_' % self.sat_sensor + str(self.ymd)
p.title = u'dcc： ' + str(title_name) + u' (分辨率1度)'
opath_fig = os.path.join(self.ofile, 'Daily', '%s' % self.ymd)
if not os.path.exists(opath_fig):
os.makedirs(opath_fig)
fig_name = os.path.join(
opath_fig, '%s_%s_dcc_daily.png' % (self.sat_sensor, self.ymd))
p.savefig(fig_name)
opath_hdf = os.path.join(self.ofile, 'Daily', '%s' % self.ymd)
if not os.path.exists(opath_hdf):
os.mkdir(opath_hdf)
opath_hdf = os.path.join(
opath_hdf, '%s_%s_dcc_daily.hdf' % (self.sat_sensor, self.ymd))
h5file_W = h5py.File(opath_hdf, 'w')
h5file_W.create_dataset('proj_data_nums',
dtype='i2',
data=proj_data_num,
compression='gzip',
compression_opts=5,
shuffle=True)
h5file_W.close()
###########################################
# read hdf file and cont values
print("write txt: %s" % self.ymd)
if not os.path.exists(self.ofile_txt):
os.mkdir(self.ofile_txt)
opath_txt = os.path.join(
self.ofile_txt, '%s_dcc_daily_count.txt' % self.sat_sensor)
if self.ifile and len(self.ymd) == 8:
hdf = h5py.File(opath_hdf, 'r')
data_mat = hdf.get("proj_data_nums")[:]
count_value = np.sum(data_mat)
self.data_count = '%8s\t%8s\n' % (self.ymd, count_value)
lock.acquire()
self.write_txt(opath_txt)
lock.release()
def run(pair, ymd):
'''
pair: sat1+sensor1_sat2+sensor2
ymd: str YYYYMMDD
'''
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
if 'FY2' in part1 or 'FY4' in part1:
ReadMatchNC = ReadMatchNC_FY2
Type = "GEOLEO"
elif 'FY3' in part1:
ReadMatchNC = ReadMatchNC_FY3
Type = "LEOLEO"
else:
return
# load yaml
plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2))
plt_cfg = pb_io.loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
return
# Day_Night must be in 'all', 'day', 'night'
Day_Night = ['all', 'day', 'night'] # default
if "day_night" in plt_cfg.keys():
Day_Night = plt_cfg['day_night']
for each in Day_Night:
if each not in ['all', 'day', 'night']:
Day_Night.remove(each)
Bias = []
if "bias" in plt_cfg.keys():
Bias = plt_cfg["bias"]
PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数
ym = ymd[:6]
ymd = ym + '%02d' % PERIOD # 当月最后一天
Log.info(u"----- Start Drawing Monthly TBBias Analysis Pic, PAIR: {}, YMD: {}" \
u" -----".format(pair, ymd))
for idx, chan in enumerate(plt_cfg['chan']):
Log.info(u"Start Drawing Channel {}".format(chan))
oneNC = ReadMatchNC()
# load Matched NC
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.NC' % (Type, pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, nc_name)
if not os.path.isfile(filefullpath):
Log.info(u"NC not found: {}".format(filefullpath))
num_file -= 1
continue
if oneNC.LoadData(filefullpath, chan) == False:
Log.error('Error occur when reading %s of %s' %
(chan, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
continue
elif num_file != PERIOD:
Log.info(u"{} of {} file(s) found.".format(num_file, PERIOD))
# 输出目录
cur_path = os.path.join(MBA_DIR, pair, ymd[:6])
# find out day and night
if ('day' in Day_Night
or 'night' in Day_Night) and len(oneNC.time) > 0:
jd = oneNC.time / 24. / 3600. # jday from 1993/01/01 00:00:00
hour = ((jd - jd.astype('int8')) * 24).astype('int8')
day_index = (hour < 10) # utc hour<10 is day
night_index = np.logical_not(day_index)
else:
day_index = None
night_index = None
for each in Bias:
# get threhold, unit, names...
xname, yname = each, each
# xname_l = plt_cfg[xname]['name']
xname_l = xname.upper()
xunit = plt_cfg[xname]['unit']
xlimit = plt_cfg[xname]['thhold'][idx]
xmin, xmax = xlimit.split('-')
xmin = int(xmin)
xmax = int(xmax)
# get x
dset_name = xname + "1"
if hasattr(oneNC, dset_name):
x = getattr(oneNC, dset_name)
else:
Log.error("Can't plot, no %s in NC class" % dset_name)
continue
# get y
dset_name = yname + "2"
if hasattr(oneNC, dset_name):
y = getattr(oneNC, dset_name)
else:
Log.error("Can't plot, no %s in NC class" % dset_name)
continue
if 'rad' == each:
o_name = 'RadBiasMonthStats'
elif 'tbb' == each:
o_name = 'TBBiasMonthStats'
elif 'ref' == each:
o_name = 'RefBiasMonthStats'
else:
o_name = 'DUMMY'
if x.size < 10:
Log.error("Not enough match point to draw.")
continue
# rad-specified regression starts
reference_list = []
if "reference" in plt_cfg[each]:
reference_list = plt_cfg[each]['reference'][idx]
if 'all' in Day_Night:
o_file = os.path.join(
cur_path, '%s_%s_%s_ALL_%s' % (pair, o_name, chan, ym))
plot(x, y, o_file, part1, part2, chan, ym, 'ALL',
reference_list, xname, xname_l, xunit, xmin, xmax)
# ------- day ----------
if 'day' in Day_Night:
if day_index is not None and np.where(day_index)[0].size > 10:
# rad-specified
o_file = os.path.join(
cur_path, '%s_%s_%s_Day_%s' % (pair, o_name, chan, ym))
x_d = x[day_index]
y_d = y[day_index]
plot(x_d, y_d, o_file, part1, part2, chan, ym, 'Day',
reference_list, xname, xname_l, xunit, xmin, xmax)
if 'night' in Day_Night:
# ---------night ------------
if night_index is not None and np.where(
night_index)[0].size > 10:
# rad-specified
o_file = os.path.join(
cur_path,
'%s_%s_%s_Night_%s' % (pair, o_name, chan, ym))
x_n = x[night_index]
y_n = y[night_index]
plot(x_n, y_n, o_file, part1, part2, chan, ym, 'Night',
reference_list, xname, xname_l, xunit, xmin, xmax)
def run(pair, ymd, isMonthly):
'''
pair: sat1+sensor1_sat2+sensor2
ymd: str YYYYMMDD
'''
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
if 'FY2' in part1 or 'FY4' in part1:
ReadMatchNC = ReadMatchNC_FY2
Type = "GEOLEO"
elif 'FY3' in part1:
ReadMatchNC = ReadMatchNC_FY3
Type = "LEOLEO"
else:
return
# load yaml config file
plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2))
plt_cfg = pb_io.loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
return
if isMonthly:
PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数
ymd = ymd[:6] + '%02d' % PERIOD # 当月最后一天
else:
PERIOD = plt_cfg['days']
# must be in 'all', 'day', 'night'
Day_Night = ['all', 'day', 'night']
if "day_night" in plt_cfg.keys():
Day_Night = plt_cfg['day_night']
for each in Day_Night:
if each not in ['all', 'day', 'night']:
Day_Night.remove(each)
Log.info(u"----- Start Drawing Regression-Pic, PAIR: {}, YMD: {}, PERIOD: " \
u"{} -----".format(pair, ymd, PERIOD))
dict_cabr = {}
dict_cabr_d = {}
dict_cabr_n = {}
for idx, chan in enumerate(plt_cfg['chan']):
Log.info(u"Start Drawing Channel {}".format(chan))
oneNC = ReadMatchNC()
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.NC' % (Type, pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, nc_name)
if not os.path.isfile(filefullpath):
Log.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if oneNC.LoadData(filefullpath, chan) == False:
Log.error('Error occur when reading %s of %s' % (chan, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
continue
elif num_file != PERIOD:
Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD))
if isMonthly:
str_time = ymd[:6]
cur_path = os.path.join(MRA_DIR, pair, str_time)
else:
str_time = ymd
cur_path = os.path.join(DRA_DIR, pair, str_time)
# delete 0 in std
if len(oneNC.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值，而导致权重极大，所以 std>0 改成 std>0.0001
deletezeros = np.where(oneNC.rad1_std > 0.0001)
oneNC.rad1_std = oneNC.rad1_std[deletezeros]
oneNC.rad1 = oneNC.rad1[deletezeros] if len(oneNC.rad1) > 0 else oneNC.rad1
oneNC.rad2 = oneNC.rad2[deletezeros] if len(oneNC.rad2) > 0 else oneNC.rad2
oneNC.tbb1 = oneNC.tbb1[deletezeros] if len(oneNC.tbb1) > 0 else oneNC.tbb1
oneNC.tbb2 = oneNC.tbb2[deletezeros] if len(oneNC.tbb2) > 0 else oneNC.tbb2
oneNC.time = oneNC.time[deletezeros] if len(oneNC.time) > 0 else oneNC.time
if len(oneNC.ref1_std) > 0.0001:
deletezeros = np.where(oneNC.ref1_std > 0.0001)
oneNC.ref1_std = oneNC.ref1_std[deletezeros]
oneNC.ref1 = oneNC.ref1[deletezeros] if len(oneNC.ref1) > 0 else oneNC.ref1
oneNC.ref2 = oneNC.ref2[deletezeros] if len(oneNC.ref2) > 0 else oneNC.ref2
# find out day and night
if ('day' in Day_Night or 'night' in Day_Night) and len(oneNC.time) > 0:
jd = oneNC.time / 24. / 3600. # jday from 1993/01/01 00:00:00
hour = ((jd - jd.astype('int8')) * 24).astype('int8')
day_index = (hour < 10) # utc hour<10 is day
night_index = np.logical_not(day_index)
else:
day_index = None
night_index = None
for each in plt_cfg['plot']:
# get threhold, unit, names...
xname, yname = each.split('-')
xname_l = plt_cfg[xname]['name']
xunit = plt_cfg[xname]['unit']
xlimit = plt_cfg[xname]['thhold'][idx]
xmin, xmax = xlimit.split('-')
xmin = float(xmin)
xmax = float(xmax)
yname_l = plt_cfg[yname]['name']
yunit = plt_cfg[yname]['unit']
ylimit = plt_cfg[yname]['thhold'][idx]
ymin, ymax = ylimit.split('-')
ymin = float(ymin)
ymax = float(ymax)
weight = None
if 'rad' in xname:
x = oneNC.rad1
elif 'tbb' in xname:
x = oneNC.tbb1
elif 'ref' in xname:
x = oneNC.ref1
elif 'dn' in xname:
x = oneNC.dn1
else:
Log.error("Can't plot %s" % each)
continue
if 'rad' in yname:
y = oneNC.rad2
elif 'tbb' in yname:
y = oneNC.tbb2
elif 'ref' in yname:
y = oneNC.ref2
else:
Log.error("Can't plot %s" % each)
continue
if 'rad' in xname and 'rad' in yname:
if len(oneNC.rad1_std) > 0:
weight = oneNC.rad1_std
o_name = 'RadCalCoeff'
elif 'tbb' in xname and 'tbb' in yname:
o_name = 'TBBCalCoeff'
elif 'ref' in xname and 'ref' in yname:
if len(oneNC.ref1_std) > 0:
weight = oneNC.ref1_std
o_name = 'CorrcCoeff'
elif 'dn' in xname and 'ref' in yname:
o_name = 'CalCoeff'
# 画对角线
if xname == yname:
diagonal = True
else:
diagonal = False
if 'all' in Day_Night and o_name not in dict_cabr:
dict_cabr[o_name] = {}
if 'day' in Day_Night and o_name not in dict_cabr_d:
dict_cabr_d[o_name] = {}
if 'night' in Day_Night and o_name not in dict_cabr_n:
dict_cabr_n[o_name] = {}
if x.size < 10:
Log.error("Not enough match point to draw.")
if 'all' in Day_Night:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if 'day' in Day_Night:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
if 'night' in Day_Night:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
continue
# regression starts
if 'all' in Day_Night:
o_file = os.path.join(cur_path,
'%s_%s_%s_ALL_%s' % (pair, o_name, chan, str_time))
abr = plot(x, y, weight, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax, diagonal)
if abr:
dict_cabr[o_name][chan] = abr
else:
dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
# ------- day ----------
if 'day' in Day_Night:
if day_index is not None and np.where(day_index)[0].size > 10 :
#
o_file = os.path.join(cur_path,
'%s_%s_%s_Day_%s' % (pair, o_name, chan, str_time))
x_d = x[day_index]
y_d = y[day_index]
w_d = weight[day_index] if weight is not None else None
abr = plot(x_d, y_d, w_d, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax, diagonal)
if abr:
dict_cabr_d[o_name][chan] = abr
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
else:
dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
# ---------night ------------
if 'night' in Day_Night:
if night_index is not None and np.where(night_index)[0].size > 10 :
#
o_file = os.path.join(cur_path,
'%s_%s_%s_Night_%s' % (pair, o_name, chan, str_time))
x_n = x[night_index]
y_n = y[night_index]
w_n = weight[night_index] if weight is not None else None
abr = plot(x_n, y_n, w_n, o_file,
num_file, part1, part2, chan, str_time,
xname, xname_l, xunit, xmin, xmax,
yname, yname_l, yunit, ymin, ymax, diagonal)
if abr:
dict_cabr_n[o_name][chan] = abr
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
else:
dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN]
oneNC.clear()
# write txt
lock.acquire()
if 'all' in Day_Night:
for o_name in dict_cabr:
writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr, 'ALL', isMonthly)
if 'day' in Day_Night:
for o_name in dict_cabr_d:
writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr_d, 'Day', isMonthly)
if 'night' in Day_Night:
for o_name in dict_cabr_n:
writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr_n, 'Night', isMonthly)
lock.release()
def run(pair, ymd):
'''
pair: sat1+sensor1_sat2+sensor2
ymd: YYYYMMDD
'''
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
# TODO: for now only support FY2 FY4
if 'FY2' in part1 or 'FY4' in part1:
pass
else:
return
# load yaml config file
plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2))
plt_cfg = pb_io.loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
return
PERIOD = plt_cfg['days']
Log.info(u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: " \
u"{} -----".format(pair, ymd, PERIOD))
fy2_orbit = Sat_Orbit(inCfg['SAT_S2L'][sat1], ymd, ORBIT_DIR)
if fy2_orbit.error:
return
fy2_orbit.get_orbit(ymd)
if fy2_orbit.error:
return
fy2_lon = fy2_orbit.orbit['Lon'][0]
fy2_lat = fy2_orbit.orbit['Lat'][0]
if len(plt_cfg['chan']) > 3:
chan = plt_cfg['chan'][2] # FY2 IR3 has the most matchpoint
else:
chan = plt_cfg['chan'][0]
NC = ReadMatchNC()
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
filename = 'COLLOC+GEOLEOIR,%s_C_BABJ_%s.NC' % (pair, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, filename)
if not os.path.isfile(filefullpath):
Log.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if NC.LoadData(filefullpath, chan, PERIOD - daydelta) == False:
Log.error('Error occur when reading %s of %s' % (chan, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
return
elif num_file != PERIOD:
Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD))
cur_path = os.path.join(DMS_DIR, pair, ymd)
# FY2F+VISSR-MetopA+IASI_MatchedPoints_ALL_20150226.png
# find out day and night
jd = NC.time / 24. / 3600. # jday from 1993/01/01 00:00:00
hour = ((jd - jd.astype('int8')) * 24).astype('int8')
day_index = (hour < 10) # utc hour<10 is day
night_index = np.logical_not(day_index)
x = NC.lon
y = NC.lat
d = NC.days.astype('uint8') # trans to int
if len(d) == 0:
Log.error('No days info in NC.')
return
# ------- day ----------
if np.where(day_index)[0].size > 0 :
o_file = os.path.join(cur_path,
'%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd))
x_d = x[day_index]
y_d = y[day_index]
d_d = d[day_index]
draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x_d, y_d, d_d, o_file)
# ---------night ------------
if np.where(night_index)[0].size > 0 :
o_file = os.path.join(cur_path,
'%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd))
x_n = x[night_index]
y_n = y[night_index]
d_n = d[night_index]
draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x_n, y_n, d_n, o_file)
# ------- ALL ----------
if np.where(day_index)[0].size != 0 and np.where(night_index)[0].size != 0:
o_file = os.path.join(cur_path,
'%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd))
draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x, y, d, o_file)
Log.info(u"Success")
def run(pair, ymd):
'''
pair: sat1+sensor1_sat2+sensor2
ymd: YYYYMMDD
'''
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
# # TODO: for now only support FY3
if 'FY3' in part1:
Type = "LEOLEO"
else:
return
# load yaml config file
plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2))
plt_cfg = pb_io.loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
return
# PERIOD = plt_cfg['days']
PERIOD = 1
Log.info(u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: " \
u"{} -----".format(pair, ymd, PERIOD))
chan = plt_cfg['chan'][0]
NC = ReadMatchNC()
num_file = PERIOD
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
filename = "COLLOC+%sIR,%s+%s_%s+%s_C_BABJ_%s.NC" % (
Type, sat1, sensor1, sat2, sensor2, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, filename)
if not os.path.isfile(filefullpath):
Log.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if NC.LoadData(filefullpath, chan) == False:
Log.error('Error occur when reading %s of %s' %
(chan, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
return
elif num_file != PERIOD:
Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD))
cur_path = os.path.join(DMS_DIR, pair, ymd)
o_file = os.path.join(cur_path,
'%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd))
# FY2F+VISSR-MetopA+IASI_MatchedPoints_ALL_20150226.png
# find out day and night
jd = NC.time / 24. / 3600. # jday from 1993/01/01 00:00:00
hour = ((jd - jd.astype('int8')) * 24).astype('int8')
day_index = (hour < 10) # utc hour<10 is day
night_index = np.logical_not(day_index)
x = NC.lon
y = NC.lat
# d = NC.days.astype('uint8') # trans to int
# if len(d) == 0:
# Log.error('No days info in NC.')
# return
draw_butterfly(part1, part2, cur_ymd, ymd, x, y, o_file)
# ------- day ----------
if np.where(day_index)[0].size > 0:
o_file = os.path.join(
cur_path, '%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd))
x_d = x[day_index]
y_d = y[day_index]
# d_d = d[day_index]
draw_butterfly(part1, part2, cur_ymd, ymd, x_d, y_d, o_file)
# ---------night ------------
if np.where(night_index)[0].size > 0:
o_file = os.path.join(
cur_path, '%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd))
x_n = x[night_index]
y_n = y[night_index]
# d_n = d[night_index]
draw_butterfly(part1, part2, cur_ymd, ymd, x_n, y_n, o_file)
Log.info(u"Success")
def run(pair, ymd):
"""
pair: sat1+sensor1_sat2+sensor2
ymd: YYYYMMDD
"""
# 提取参数中的卫星信息和传感器信息
part1, part2 = pair.split("_")
sat1, sensor1 = part1.split("+")
sat2, sensor2 = part2.split("+")
# 判断是静止卫星还是动态卫星
if "FY2" in part1 or "FY4" in part1:
Type = "GEOLEO"
elif "FY3" in part1:
Type = "LEOLEO"
else:
Log.error("Cant distinguish the satellite type")
return
# 加载绘图配置文件
plt_cfg_file = os.path.join(MainPath, "%s_%s_3d.yaml" % (sensor1, sensor2))
plt_cfg = loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
Log.error("Not find the config file: {}".format(plt_cfg_file))
return
# 读取配置文件的信息
PERIOD = plt_cfg['collocation_map']['days'] # 回滚天数
chans = plt_cfg['collocation_map']['chan'] # 通道
maptype = plt_cfg['collocation_map']['maptype'] # 需要绘制的类型
if 'area' in maptype: # 区域块视图
area = plt_cfg['collocation_map']['area']
else:
area = None
if 'polar' in maptype: # 两极视图
polar = plt_cfg['collocation_map']['polar']
else:
polar = None
# 读取范围配置
if not area and not polar:
return
else:
map_range = (polar, area)
Log.info(
u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: {} -----"
.format(pair, ymd, PERIOD))
# 读取 HDF5 文件数据
oneHDF5 = ReadHDF5()
num_file = PERIOD
cur_ymd = pb_time.ymd_plus(ymd, 1) # 回滚天数，现在为 1
for daydelta in xrange(PERIOD):
cur_ymd = pb_time.ymd_plus(ymd, -daydelta)
filename = "COLLOC+%sIR,%s+%s_%s+%s_C_BABJ_%s.hdf5" % (
Type, sat1, sensor1, sat2, sensor2, cur_ymd)
filefullpath = os.path.join(MATCH_DIR, pair, filename)
if not os.path.isfile(filefullpath):
Log.info(u"File not found: {}".format(filefullpath))
num_file -= 1
continue
if not oneHDF5.LoadData(filefullpath, chans):
Log.error('Error occur when reading %s of %s' %
(chans, filefullpath))
if num_file == 0:
Log.error(u"No file found.")
return
elif num_file != PERIOD:
Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD))
cur_path = os.path.join(DMS_DIR, pair, ymd)
o_file = os.path.join(cur_path,
'%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd))
# find out day and night
vect_is_day = np.vectorize(is_day_timestamp_and_lon)
day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1)
night_index = np.logical_not(day_index)
x = oneHDF5.lon1 # 经度数据
y = oneHDF5.lat1 # 维度数据
print 'date: {}, x_all: {} y_all: {} '.format(ymd, len(x), len(y))
draw_butterfly(part1, part2, cur_ymd, ymd, x, y, o_file, map_range)
# ------- day ----------
if np.where(day_index)[0].size > 0:
o_file = os.path.join(
cur_path, '%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd))
x_d = x[day_index]
y_d = y[day_index]
print 'date: {}, x_day: {} y_day: {} '.format(ymd, len(x_d), len(y_d))
draw_butterfly(part1, part2, cur_ymd, ymd, x_d, y_d, o_file, map_range)
# ---------night ------------
if np.where(night_index)[0].size > 0:
o_file = os.path.join(
cur_path, '%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd))
x_n = x[night_index]
y_n = y[night_index]
print 'date: {}, x_night: {} y_night: {} '.format(
ymd, len(x_n), len(y_n))
draw_butterfly(part1, part2, cur_ymd, ymd, x_n, y_n, o_file, map_range)
Log.info(u"Success")
def runLEO_LEO_SNOX(s1, s2, sat_dist, timeGap, outPath, Log, day_counts=1):
'''
极轨对极轨 交叉点计算
'''
zangle = 180 # 天顶角阈值 80改为180 2018/6/15
drew_points = []
output_list = []
for day_num in range(0, day_counts):
ymd = ymd_plus(s1.ymd, day_num)
s1.get_orbit(ymd)
if s1.error:
continue
s2.get_orbit(ymd)
if s2.error:
continue
if len(s1.orbit) == 0 or len(s2.orbit) == 0:
continue
s1.divide_by_lat0()
s2.divide_by_lat0()
day_list = [] # 输出用
for i in xrange(len(s1.divide)):
index1 = s1.divide[i]
for j in xrange(len(s2.divide)):
index2 = s2.divide[j]
timeDiff = index2 - index1
if abs(timeDiff) > timeGap * 60:
continue
dist = distance_GreatCircle(s1.orbit['Lat'][index1],
s1.orbit['Lon'][index1],
s2.orbit['Lat'][index2],
s2.orbit['Lon'][index2])
if dist > sat_dist:
continue
# TODO: 只保留太阳天顶角<80°的点（白天)
yy, mm, dd = s1.orbit['date'][index1].split(".")
hh = int(s1.orbit['time'][index1][0:2])
sz = solar_zen(yy, mm, dd, hh, s1.orbit['Lon'][index1],
s1.orbit['Lat'][index1])
if sz > zangle:
continue
day_list.append([
s1.orbit['date'][index1].replace('.', ''),
s1.orbit['time'][index1][0:8], s1.orbit['Lat'][index1],
s1.orbit['Lon'][index1], s2.orbit['time'][index2][0:8],
s2.orbit['Lat'][index2], s2.orbit['Lon'][index2], dist,
timeDiff
])
# 画图所需信息
st = max(index1 - 240, 0)
et = min(index1 + 80, len(s1.orbit['Lon']))
trails1 = [s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et]]
st = max(index2 - 240, 0)
et = min(index2 + 80, len(s2.orbit['Lon']))
trails2 = [s2.orbit['Lon'][st:et], s2.orbit['Lat'][st:et]]
drew_points.append(
(s1.orbit['Lon'][index1], s1.orbit['Lat'][index1],
s2.orbit['Lon'][index2], s2.orbit['Lat'][index2], trails1,
trails2))
output_list.extend(day_list)
# -----------------------
if len(output_list) == 0:
Log.info('No SNOX!')
return
wout_fig_file = outPath.replace('.txt', '_worldmap.png')
ymd_s = ymd2y_m_d(s1.ymd)
# 画交叉点图像
if len(drew_points) > 0:
draw_world(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points,
wout_fig_file)
folder = os.path.dirname(outPath)
if not os.path.isdir(folder):
os.makedirs(folder)
# 输出交叉点文件
fid_Result = open(outPath, 'w')
# 写十行信息
fid_Result.write('Sat1: %s \n' % (s1.sat))
fid_Result.write('Sat2: %s \n' % (s2.sat))
if ymd_s == ymd:
fid_Result.write('Time: %s \n' % ymd_s)
else:
fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd))
fid_Result.write('Time Gap MAX = %d min\n' % timeGap)
fid_Result.write('Dist MAX = %d (km)\n' % sat_dist)
fid_Result.write('Solar Zenith MAX = %s (deg)\n' % str(zangle))
fid_Result.write('Calc time : %s \n' %
get_local_time().strftime('%Y.%m.%d %H:%M:%S'))
fid_Result.write('\n')
title_line = 'YMD HMS(%s) Lat,Lon(%s) HMS(%s) Lat,Lon(%s) Distance(km) Time_Diff(sec)\n' % \
(s1.sat, s1.sat, s2.sat, s2.sat)
fid_Result.write(title_line)
fid_Result.write('-' * len(title_line) + '\n')
for eachline in output_list:
fid_Result.write(
'%s %s %6.2f %-7.2f %s %6.2f %-7.2f %7.2f %4d\n'
% tuple(eachline))
fid_Result.close()
s1.clear()
Log.info('Success')
def runLEO_LEO(s1,
s2,
sat_dist,
timeGap_high,
timeGap_low,
outPath,
Log,
day_counts=1):
'''
极轨对极轨 交叉点计算
'''
drew_points = []
output_list = []
for day_num in range(0, day_counts):
ymd = ymd_plus(s1.ymd, day_num)
s1.get_orbit(ymd)
if s1.error:
continue
s2.get_orbit(ymd)
if s2.error:
continue
if len(s1.orbit) == 0 or len(s2.orbit) == 0:
continue
s1.divide_orbit()
s2.divide_orbit()
day_list = [] # 输出用
# day_points = [] # 画图用
for i in xrange(len(s1.divide)):
if i == 0:
continue
ii_s = s1.divide[i - 1]
ii_e = s1.divide[i]
if abs(ii_e - ii_s) < 2:
continue
line1 = LineString(
zip(s1.orbit['Lon'][ii_s:ii_e], s1.orbit['Lat'][ii_s:ii_e]))
for j in xrange(len(s2.divide)):
if j == 0:
continue
jj_s = s2.divide[j - 1]
jj_e = s2.divide[j]
if abs(jj_e - jj_s) < 2:
continue
if abs(ii_s + ii_e - jj_s -
jj_e) > timeGap_low * 60 * 2 * 4: # 中间点4倍时间范围粗筛
continue
line2 = LineString(
zip(s2.orbit['Lon'][jj_s:jj_e],
s2.orbit['Lat'][jj_s:jj_e]))
inters = line1.intersection(line2)
if inters.geom_type == 'Point':
pass
elif inters.geom_type == 'MultiPoint':
inters = np.array(inters)
if (np.abs(np.diff(inters[:, 0])) < 1.).all() and \
(np.abs(np.diff(inters[:, 1])) < 1.).all():
inters = [np.mean(inters[:, 0]), np.mean(inters[:, 1])]
else:
continue
else:
continue
inters = np.array(inters) # 转成array
if len(inters) > 0:
index1 = ii_s + \
getIndex(s1.orbit['Lon'][ii_s:ii_e], inters[0])
index2 = jj_s + \
getIndex(s2.orbit['Lon'][jj_s:jj_e], inters[0])
timeDiff = index2 - index1
if abs(timeDiff) <= timeGap_low * 60:
# 60度以上高纬 timeGap过滤
if abs(s1.orbit['Lat'][index1]) > 60. and \
abs(timeDiff) > timeGap_high * 60:
continue
dd = distance_GreatCircle(s1.orbit['Lat'][index1],
s1.orbit['Lon'][index1],
s2.orbit['Lat'][index2],
s2.orbit['Lon'][index2])
if dd > sat_dist:
continue
day_list.append([
s1.orbit['date'][index1].replace('.', ''),
s1.orbit['time'][index1][0:8],
s1.orbit['Lat'][index1], s1.orbit['Lon'][index1],
s2.orbit['time'][index2][0:8],
s2.orbit['Lat'][index2], s2.orbit['Lon'][index2],
dd, timeDiff
])
# 画图所需信息
st = max(index1 - 240, 0)
et = min(index1 + 80, len(s1.orbit['Lon']))
trails1 = [
s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et]
]
st = max(index2 - 240, 0)
et = min(index2 + 80, len(s2.orbit['Lon']))
trails2 = [
s2.orbit['Lon'][st:et], s2.orbit['Lat'][st:et]
]
drew_points.append(
(s1.orbit['Lon'][index1], s1.orbit['Lat'][index1],
s2.orbit['Lon'][index2], s2.orbit['Lat'][index2],
trails1, trails2))
# ------- 交叉点多 隔3个 取一个 --------
output_list.extend(day_list[::3])
# drew_points.extend(day_points[::3])
# -----------------------
if len(output_list) == 0:
# 20160708 黄叶建huangyj modify
Log.info('No SNO!')
return
pout_fig_file = outPath.replace('.txt', '_polarmap.png')
wout_fig_file = outPath.replace('.txt', '_worldmap.png')
ymd_s = ymd2y_m_d(s1.ymd)
# 画交叉点图像
if len(drew_points) > 0:
draw_polar(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points,
pout_fig_file)
draw_world(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points,
wout_fig_file)
folder = os.path.dirname(outPath)
if not os.path.isdir(folder):
os.makedirs(folder)
# 输出交叉点文件
fid_Result = open(outPath, 'w')
fid_Result.write('Sat1: %s \n' % (s1.sat))
fid_Result.write('Sat2: %s \n' % (s2.sat))
if ymd_s == ymd:
fid_Result.write('Time: %s \n' % ymd_s)
else:
fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd))
fid_Result.write('When |lat|<=60: Time Gap MAX = %d min\n' % timeGap_low)
fid_Result.write('When |lat|>60: Time Gap MAX = %d min\n' % timeGap_high)
fid_Result.write('Dist MAX = %d (km)\n' % sat_dist)
fid_Result.write('Calc time : %s \n' %
get_local_time().strftime('%Y.%m.%d %H:%M:%S'))
fid_Result.write('\n')
title_line = 'YMD HMS(%s) Lat,Lon(%s) HMS(%s) Lat,Lon(%s) Distance(km) Time_Diff(sec)\n' % \
(s1.sat, s1.sat, s2.sat, s2.sat)
fid_Result.write(title_line)
fid_Result.write('-' * len(title_line) + '\n')
for eachline in output_list:
fid_Result.write(
'%s %s %6.2f %-7.2f %s %6.2f %-7.2f %7.2f %4d\n'
% tuple(eachline))
fid_Result.close()
s1.clear()
Log.info('Success')
def runSatPassingFixedPoint(s1,
fix_list,
fix_dist,
outPath,
FIX_DICT,
Log,
day_counts=1):
'''
s1 : Sat_Orbit的实例
fix_list: 固定点组名list
fix_dist: 距离阈值
outPath： 输出txt全路径
day_counts： 算几天
'''
out_fig = outPath.replace('.txt', '.png')
fixDict = {}
drew_points = []
output_list = []
# 去重
for fix_group in fix_list: # 循环不同组
for fix_point in FIX_DICT[fix_group].keys(): # 循环每个组中的键
if fix_point not in fixDict.keys(): # 如果键不存在字典中，则将该键和键值进行存入
fixDict[fix_point] = FIX_DICT[fix_group][fix_point]
for fix_point in fixDict.keys():
fixed_lon, fixed_lat = [float(e)
for e in fixDict[fix_point]] # 提取站点的经度和维度
# log.debug('%s and %s of %s' % (fixed_point[0], sat2, date))
for day_num in range(0, day_counts):
ymd = ymd_plus(s1.ymd, day_num) # 将进行进行进行往后推
s1.get_orbit(ymd)
if s1.error:
continue
if len(s1.orbit) == 0:
continue
l = len(s1.orbit)
lat_fixed_array = np.array([fixed_lat] * l)
lon_fixed_array = np.array([fixed_lon] * l)
with warnings.catch_warnings():
warnings.simplefilter("ignore") # Suppressing RuntimeWarning
deltaDist_list = distance_GreatCircle_np(
lat_fixed_array, lon_fixed_array, s1.orbit['Lat'],
s1.orbit['Lon'])
y_growth_flips = np.where(
np.diff(np.diff(deltaDist_list) > 0))[0] + 1
i_old = 0
cross_list = []
for i in y_growth_flips:
if (deltaDist_list[i] < fix_dist and (i - i_old > 10)
and # 去掉间隔秒数在10秒以内的点
(deltaDist_list[i] - deltaDist_list[i_old] < 0)):
# 每行2个元素：index, deltaDist_list
cross_list.append((i, deltaDist_list[i]))
i_old = i
# 排序 按deltaDist从小到大
cross_list.sort(cmp_byDeltaDist)
if (len(cross_list) == 0):
continue
j = 0
for each in cross_list:
cross_index = each[0]
# TODO:过滤夜晚数据 -------------------
# 以太阳天顶角 大于85度作为夜晚的判断。
# 用这种方法，'Greenland' 'Dome_C'就不需要特别考虑极夜极昼了
ymd_sol = s1.orbit['date'][cross_index]
yy = int(ymd_sol[:4])
mm = int(ymd_sol[5:7])
dd = int(ymd_sol[8:10])
hh = int(s1.orbit['time'][cross_index][0:2])
if solar_zen(yy, mm, dd, hh, s1.orbit['Lon'][cross_index],
s1.orbit['Lat'][cross_index]) >= 85.:
continue
# ------------------------------
output_list.append([
s1.orbit['date'][cross_index].replace('.', ''),
s1.orbit['time'][cross_index][0:8], fix_point, fixed_lat,
fixed_lon, s1.orbit['Lat'][cross_index],
s1.orbit['Lon'][cross_index], deltaDist_list[cross_index]
])
# 保留画图信息
st = max(cross_index - 240, 0)
et = min(cross_index + 80, len(s1.orbit['Lon']))
drew_points.append(
(fixed_lon, fixed_lat, s1.orbit['Lon'][cross_index],
s1.orbit['Lat'][cross_index], [None, None],
[s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et]]))
j = j + 1
if fix_point in ['Greenland', 'Dome_C']:
pass # 冰雪目标全部保留
elif j >= 1: # 其他目标只保留最近点
break
if (len(output_list) == 0):
Log.info('No SNO!')
return
# 排序
output_list.sort(cmp_col0_col1)
ymd_s = ymd2y_m_d(s1.ymd)
# 画图
if len(drew_points) > 0:
draw_fixed(s1.sat, ymd_s, ymd2y_m_d(ymd), drew_points, out_fig)
folder = os.path.dirname(outPath)
if not os.path.isdir(folder):
os.makedirs(folder)
# 写文件
fid_Result = open(outPath, 'w')
fid_Result.write('Sat: %s \n' % s1.sat)
if ymd_s == ymd:
fid_Result.write('Time: %s \n' % ymd_s)
else:
fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd))
fid_Result.write('Dist MAX = %s (km) \n' % fix_dist)
fid_Result.write(u'Solar Zenith MAX = 85° \n')
fid_Result.write(u' 对于Greenland和Dome_C，保留所有符合阈值的每轨最近点\n')
fid_Result.write(u' 对于其它固定点，只保留符合阈值的全天最近点\n')
fid_Result.write('Calc time : %s \n' %
get_local_time().strftime('%Y.%m.%d %H:%M:%S'))
fid_Result.write('\n')
title_line = 'YMD HMS(%s) %-24s %-16s %-19s %s \n' % \
(s1.sat, 'Fixed_Point', 'Lat,Lon(Fixed)',
'Lat,Lon(%s)' % s1.sat, 'Distance(km)')
fid_Result.write(title_line)
fid_Result.write('-' * len(title_line) + '\n')
for eachline in output_list:
fid_Result.write(
'%s %s %-24s %6.2f %-7.2f %6.2f %-7.2f %7.2f\n' %
tuple(eachline))
fid_Result.close()
# clean
s1.clear()
Log.info('Success')