def product_fy4a_1km_disk_full_data_orbit(date_start=None,
date_end=None,
thread=THREAD,
**kwargs):
"""
绘制原始4KM数据的图像
3个产品，每个产品2张图像，共6张图像
:param date_start: 开始日期 datetime
:param date_end: 结束日期 datetime
:param thread:
:return:
"""
resultid_in = 'FY4A_AGRI_L2_SSI_Orbit'
resultid_out = 'FY4A_AGRI_L2_SSI_Orbit'
frequency = 'Orbit'
resolution_type_in = '4KMCorrect'
resolution_type = '1KM'
out_dir = os.path.join(
DATA_ROOT_DIR, 'SSIData/FY4A/SSI_{resolution_type}/Full/{frequency}')
out_dir = out_dir.format(resolution_type=resolution_type,
frequency=frequency)
planid = 1
results = find_result_data(resultid=resultid_in,
datatime_start=date_start,
datatime_end=date_end,
resolution_type=resolution_type_in)
in_files = [row.address for row in results]
in_files.sort()
in_files_length = len(in_files)
print('找到的文件总数:{}'.format(in_files_length))
print('开始生产')
p = Pool(thread)
for in_file in in_files[:]:
in_file_name = os.path.basename(in_file)
datatime = FY4ASSI.get_date_time_orbit(in_file)
if datatime.minute != 0:
continue
date_time = datatime.strftime('%Y%m%d%H%M%S')
out_file_name = in_file_name.replace(resolution_type_in,
resolution_type)
out_file = os.path.join(out_dir, date_time[:8], out_file_name)
if DEBUG or thread == 1:
fy4a_ssi_4km_to_1km(in_file, out_file, resultid_out, planid,
datatime, resolution_type)
else:
p.apply_async(fy4a_ssi_4km_to_1km,
args=(in_file, out_file, resultid_out, planid,
datatime, resolution_type))
p.close()
p.join()
print('完成全部的任务:{}'.format(sys._getframe().f_code.co_name))
def itcal(in_file,
out_file,
resultid=None,
planid=None,
datatime=None,
resolution_type=None):
# 如果原来的整点数据不存在，直接使用G0进行补充
# 如果原来的整点数据存在，使用G0进行校正
area_type = 'Full_DISK'
if os.path.isfile(out_file):
print('数据已经存在: {}'.format(out_file))
if not exist_result_data(resultid=resultid,
datatime=datatime,
resolution_type=resolution_type,
area_type=area_type):
add_result_data(resultid=resultid,
planid=planid,
address=out_file,
datatime=datatime,
resolution_type=resolution_type,
area_type=area_type,
element=None)
return
print('<<< itcal: {}'.format(in_file))
beta = 35.0 # 常量
try:
datas = FY4ASSI(in_file)
except Exception as why:
print(why)
print('初始化FY4A SSI读取类错误')
return
date_time = FY4ASSI.get_date_time_orbit(in_file)
y, m, d, hr, minus = assignTime(date_time)
e = assignE(y, m, d)
doy = calDoy(y, m, d)
delta = calDelta(doy)
print(delta)
try:
lons = FY4ASSI.get_longitude_4km()
lats = FY4ASSI.get_latitude_4km()
except Exception as why:
print(why)
print('读取lons和lats错误')
return
DQF = np.ones_like(lons, dtype=np.int8) # 标识数据集
Omega = calOmega(hr, minus, lons, e)
cos_the_taz = calCosThetaz(lats, delta, Omega)
G0 = calG0(doy, cos_the_taz)
print('G0')
print(np.nanmin(G0), np.nanmax(G0))
print((G0 > 0).sum())
index_invalid_g0 = np.logical_or(
G0 >= 1500, G0 <= 0) # ########################## G0的无效值赋值为nan
G0[index_invalid_g0] = np.nan
if np.isnan(G0).all():
print('Warning::::::::没有有效的G0数据，不生产数据')
return
# G0无效
DQF[index_invalid_g0] = 0
# 校正总直散数据
if os.path.isfile(in_file):
try:
Itol = datas.get_ssi()
Ib = datas.get_dirssi()
Id = datas.get_difssi()
except Exception as why:
print(why)
print('读取ssi，dirssi和difssi错误')
return
# 将G0 >= 1400, G0 <= 0的值置为nan
Itol[index_invalid_g0] = np.nan
Ib[index_invalid_g0] = np.nan
Id[index_invalid_g0] = np.nan
# Itol 有效
index_valid_itol = np.logical_and(np.isfinite(Itol), Itol < G0)
DQF[index_valid_itol] = 1
# 校正G0有效，但是Itol无效的数据
index_invalid_itol = np.logical_or(
Itol > G0, np.logical_and(np.isnan(Itol), np.isfinite(G0)))
Itol[index_invalid_itol] = G0_Correct * G0[index_invalid_itol]
Ib[index_invalid_itol] = 0.3 * Itol[index_invalid_itol]
Id[index_invalid_itol] = 0.7 * Itol[index_invalid_itol]
DQF[index_invalid_itol] = 2
else:
Itol = G0_Correct * G0
Ib = 0.3 * Itol
Id = 0.7 * Itol
# 计算Gt和DNI
Rb = calRb(lats, beta, delta, Omega, cos_the_taz)
Ai = Ib / G0
Gt = calGt(Ib, Id, Ai, Rb, beta, Itol)
DNI = Ib / cos_the_taz
# 校正Gt
index_invalid_gt = np.logical_and(Gt < 0, np.isfinite(G0))
Gt[index_invalid_gt] = 0.75 * G0[index_invalid_gt]
DQF[index_invalid_gt] = 3
Gt[lats < 0] = np.nan # 20191121 AnNing 根据用户需求，Gt的数据只生产北半球
# 输出数据
result = {
'G0': G0,
'Gt': Gt,
'DNI': DNI,
'SSI': Itol,
'DirSSI': Ib,
'DifSSI': Id,
'DQF': DQF
}
try:
_write_out_file(out_file, result)
if os.path.isfile(out_file) and not exist_result_data(
resultid=resultid,
datatime=datatime,
resolution_type=resolution_type,
area_type=area_type):
add_result_data(resultid=resultid,
planid=planid,
address=out_file,
datatime=datatime,
resolution_type=resolution_type,
area_type=area_type,
element=None)
except Exception as why:
print(why)
print('输出结果文件错误')
return
def combine_full(in_files, out_file, day=False, resultid=None, planid=None, datatime=None, resolution_type=None):
"""
:param in_files:
:param out_file:
:param day: 是否日合成，因为日合成的时候，要改变数据的单位为KW/m2
:param resultid:
:param planid:
:param datatime:
:param resolution_type:
:return:
"""
out_path = os.path.dirname(out_file)
if not os.path.isdir(out_path):
os.makedirs(out_path)
area_type = 'Full_DISK'
if os.path.isfile(out_file):
print('文件已经存在，跳过:{}'.format(out_file))
if not exist_result_data(resultid=resultid, datatime=datatime,
resolution_type=resolution_type,
area_type=area_type):
print('开始入库')
add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime,
resolution_type=resolution_type, area_type=area_type)
return
data_all = {
'SSI': None,
'DirSSI': None,
'DifSSI': None,
'G0': None,
'Gt': None,
'DNI': None,
}
for in_file in in_files:
print('combine <<< :{}'.format(in_file))
try:
datas = FY4ASSI(in_file)
# 日合成的时候，只使用整点数据
if day:
date_time = FY4ASSI.get_date_time_orbit(in_file)
if date_time.minute != 0:
continue
data_get = {
'SSI': datas.get_ssi,
'DirSSI': datas.get_ib,
'DifSSI': datas.get_id,
'G0': datas.get_g0,
'Gt': datas.get_gt,
'DNI': datas.get_dni,
}
for dataname in data_all:
data_all[dataname] = add_data(data_all[dataname], data_get[dataname]())
except Exception as why:
print(why)
print('合成数据过程出错，文件为：{}'.format(in_file))
continue
# 从时次产品转为日产品的时候，单位变为kw/m2
try:
if day:
print('从时次产品转为日产品的时候，单位变为kw/m2')
for dataname in data_all:
data = data_all[dataname]
if data is not None:
data_all[dataname] = data * 0.001
except Exception as why:
print(why)
print('转换单位出错')
try:
_write_out_file(out_file, data_all)
if os.path.isfile(out_file) and not exist_result_data(resultid=resultid, datatime=datatime,
resolution_type=resolution_type,
area_type=area_type):
print('开始入库')
add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime,
resolution_type=resolution_type, area_type=area_type)
except Exception as why:
print(why)
print('输出结果文件错误')
return
def fy4a_save_4km_orbit_data_in_database(date_start=None,
date_end=None,
**kwargs):
print(date_start)
print(date_end)
# source_dir = os.path.join('/FY4/FY4A/AGRI/L2/SSI/DISK/NOM')
source_dir = os.path.join(
'/home/gfssi/GFData/SourceData/FY4A/SSI_4KM/Full/Orbit')
ssi_dir = os.path.join(DATA_ROOT_DIR, 'SSIData', 'FY4A', 'SSI_4KM', 'Full',
'Orbit')
ext = '.NC'
resultid = 'FY4A_AGRI_L2_SSI_Orbit'
planid = 1
results = find_result_data(resultid=resultid,
datatime_start=date_start,
datatime_end=date_end,
resolution_type='4KM')
results_files = [row.address for row in results]
results_files = set(results_files)
session = Session()
count = 0
for root, dirs, files in os.walk(source_dir):
for file_name in files:
if ext is not None:
if '.' not in ext:
ext = '.' + ext
if os.path.splitext(file_name)[1].lower() != ext.lower():
continue
try:
src_file = os.path.join(root, file_name)
datatime = FY4ASSI.get_date_time_orbit(src_file)
if date_start is not None and date_end is not None:
if not date_start <= datatime <= date_end:
continue
yyyymmdd = datatime.strftime("%Y%m%d")
dst_file = os.path.join(ssi_dir, yyyymmdd, file_name)
dst_file = dst_file.replace('4000M', '4KM')
if not os.path.isfile(dst_file):
dst_dir = os.path.dirname(dst_file)
if not os.path.isdir(dst_dir):
os.makedirs(dst_dir)
os.symlink(src_file, dst_file)
if dst_file in results_files:
continue
result_data = ResultData()
result_data.resultid = resultid
result_data.planid = planid
result_data.address = dst_file
result_data.datatime = datatime
result_data.createtime = datetime.now()
result_data.resolution_type = '4KM'
result_data.area_type = 'Full_DISK'
result_data.element = None
session.add(result_data)
count += 1
print('{} -----> {}'.format(src_file, dst_file))
if count >= 500:
session.commit()
count = 0
except Exception as why:
print(why)
os.remove(dst_file)
session.commit()
session.close()
