def product_fy4a_1km_disk_full_data_orbit(date_start=None, date_end=None, thread=THREAD, **kwargs): """ 绘制原始4KM数据的图像 3个产品,每个产品2张图像,共6张图像 :param date_start: 开始日期 datetime :param date_end: 结束日期 datetime :param thread: :return: """ resultid_in = 'FY4A_AGRI_L2_SSI_Orbit' resultid_out = 'FY4A_AGRI_L2_SSI_Orbit' frequency = 'Orbit' resolution_type_in = '4KMCorrect' resolution_type = '1KM' out_dir = os.path.join( DATA_ROOT_DIR, 'SSIData/FY4A/SSI_{resolution_type}/Full/{frequency}') out_dir = out_dir.format(resolution_type=resolution_type, frequency=frequency) planid = 1 results = find_result_data(resultid=resultid_in, datatime_start=date_start, datatime_end=date_end, resolution_type=resolution_type_in) in_files = [row.address for row in results] in_files.sort() in_files_length = len(in_files) print('找到的文件总数:{}'.format(in_files_length)) print('开始生产') p = Pool(thread) for in_file in in_files[:]: in_file_name = os.path.basename(in_file) datatime = FY4ASSI.get_date_time_orbit(in_file) if datatime.minute != 0: continue date_time = datatime.strftime('%Y%m%d%H%M%S') out_file_name = in_file_name.replace(resolution_type_in, resolution_type) out_file = os.path.join(out_dir, date_time[:8], out_file_name) if DEBUG or thread == 1: fy4a_ssi_4km_to_1km(in_file, out_file, resultid_out, planid, datatime, resolution_type) else: p.apply_async(fy4a_ssi_4km_to_1km, args=(in_file, out_file, resultid_out, planid, datatime, resolution_type)) p.close() p.join() print('完成全部的任务:{}'.format(sys._getframe().f_code.co_name))
def itcal(in_file, out_file, resultid=None, planid=None, datatime=None, resolution_type=None): # 如果原来的整点数据不存在,直接使用G0进行补充 # 如果原来的整点数据存在,使用G0进行校正 area_type = 'Full_DISK' if os.path.isfile(out_file): print('数据已经存在: {}'.format(out_file)) if not exist_result_data(resultid=resultid, datatime=datatime, resolution_type=resolution_type, area_type=area_type): add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime, resolution_type=resolution_type, area_type=area_type, element=None) return print('<<< itcal: {}'.format(in_file)) beta = 35.0 # 常量 try: datas = FY4ASSI(in_file) except Exception as why: print(why) print('初始化FY4A SSI读取类错误') return date_time = FY4ASSI.get_date_time_orbit(in_file) y, m, d, hr, minus = assignTime(date_time) e = assignE(y, m, d) doy = calDoy(y, m, d) delta = calDelta(doy) print(delta) try: lons = FY4ASSI.get_longitude_4km() lats = FY4ASSI.get_latitude_4km() except Exception as why: print(why) print('读取lons和lats错误') return DQF = np.ones_like(lons, dtype=np.int8) # 标识数据集 Omega = calOmega(hr, minus, lons, e) cos_the_taz = calCosThetaz(lats, delta, Omega) G0 = calG0(doy, cos_the_taz) print('G0') print(np.nanmin(G0), np.nanmax(G0)) print((G0 > 0).sum()) index_invalid_g0 = np.logical_or( G0 >= 1500, G0 <= 0) # ########################## G0的无效值赋值为nan G0[index_invalid_g0] = np.nan if np.isnan(G0).all(): print('Warning::::::::没有有效的G0数据,不生产数据') return # G0无效 DQF[index_invalid_g0] = 0 # 校正总直散数据 if os.path.isfile(in_file): try: Itol = datas.get_ssi() Ib = datas.get_dirssi() Id = datas.get_difssi() except Exception as why: print(why) print('读取ssi,dirssi和difssi错误') return # 将G0 >= 1400, G0 <= 0的值置为nan Itol[index_invalid_g0] = np.nan Ib[index_invalid_g0] = np.nan Id[index_invalid_g0] = np.nan # Itol 有效 index_valid_itol = np.logical_and(np.isfinite(Itol), Itol < G0) DQF[index_valid_itol] = 1 # 校正G0有效,但是Itol无效的数据 index_invalid_itol = np.logical_or( Itol > G0, np.logical_and(np.isnan(Itol), np.isfinite(G0))) Itol[index_invalid_itol] = G0_Correct * G0[index_invalid_itol] Ib[index_invalid_itol] = 0.3 * Itol[index_invalid_itol] Id[index_invalid_itol] = 0.7 * Itol[index_invalid_itol] DQF[index_invalid_itol] = 2 else: Itol = G0_Correct * G0 Ib = 0.3 * Itol Id = 0.7 * Itol # 计算Gt和DNI Rb = calRb(lats, beta, delta, Omega, cos_the_taz) Ai = Ib / G0 Gt = calGt(Ib, Id, Ai, Rb, beta, Itol) DNI = Ib / cos_the_taz # 校正Gt index_invalid_gt = np.logical_and(Gt < 0, np.isfinite(G0)) Gt[index_invalid_gt] = 0.75 * G0[index_invalid_gt] DQF[index_invalid_gt] = 3 Gt[lats < 0] = np.nan # 20191121 AnNing 根据用户需求,Gt的数据只生产北半球 # 输出数据 result = { 'G0': G0, 'Gt': Gt, 'DNI': DNI, 'SSI': Itol, 'DirSSI': Ib, 'DifSSI': Id, 'DQF': DQF } try: _write_out_file(out_file, result) if os.path.isfile(out_file) and not exist_result_data( resultid=resultid, datatime=datatime, resolution_type=resolution_type, area_type=area_type): add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime, resolution_type=resolution_type, area_type=area_type, element=None) except Exception as why: print(why) print('输出结果文件错误') return
def combine_full(in_files, out_file, day=False, resultid=None, planid=None, datatime=None, resolution_type=None): """ :param in_files: :param out_file: :param day: 是否日合成,因为日合成的时候,要改变数据的单位为KW/m2 :param resultid: :param planid: :param datatime: :param resolution_type: :return: """ out_path = os.path.dirname(out_file) if not os.path.isdir(out_path): os.makedirs(out_path) area_type = 'Full_DISK' if os.path.isfile(out_file): print('文件已经存在,跳过:{}'.format(out_file)) if not exist_result_data(resultid=resultid, datatime=datatime, resolution_type=resolution_type, area_type=area_type): print('开始入库') add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime, resolution_type=resolution_type, area_type=area_type) return data_all = { 'SSI': None, 'DirSSI': None, 'DifSSI': None, 'G0': None, 'Gt': None, 'DNI': None, } for in_file in in_files: print('combine <<< :{}'.format(in_file)) try: datas = FY4ASSI(in_file) # 日合成的时候,只使用整点数据 if day: date_time = FY4ASSI.get_date_time_orbit(in_file) if date_time.minute != 0: continue data_get = { 'SSI': datas.get_ssi, 'DirSSI': datas.get_ib, 'DifSSI': datas.get_id, 'G0': datas.get_g0, 'Gt': datas.get_gt, 'DNI': datas.get_dni, } for dataname in data_all: data_all[dataname] = add_data(data_all[dataname], data_get[dataname]()) except Exception as why: print(why) print('合成数据过程出错,文件为:{}'.format(in_file)) continue # 从时次产品转为日产品的时候,单位变为kw/m2 try: if day: print('从时次产品转为日产品的时候,单位变为kw/m2') for dataname in data_all: data = data_all[dataname] if data is not None: data_all[dataname] = data * 0.001 except Exception as why: print(why) print('转换单位出错') try: _write_out_file(out_file, data_all) if os.path.isfile(out_file) and not exist_result_data(resultid=resultid, datatime=datatime, resolution_type=resolution_type, area_type=area_type): print('开始入库') add_result_data(resultid=resultid, planid=planid, address=out_file, datatime=datatime, resolution_type=resolution_type, area_type=area_type) except Exception as why: print(why) print('输出结果文件错误') return
def fy4a_save_4km_orbit_data_in_database(date_start=None, date_end=None, **kwargs): print(date_start) print(date_end) # source_dir = os.path.join('/FY4/FY4A/AGRI/L2/SSI/DISK/NOM') source_dir = os.path.join( '/home/gfssi/GFData/SourceData/FY4A/SSI_4KM/Full/Orbit') ssi_dir = os.path.join(DATA_ROOT_DIR, 'SSIData', 'FY4A', 'SSI_4KM', 'Full', 'Orbit') ext = '.NC' resultid = 'FY4A_AGRI_L2_SSI_Orbit' planid = 1 results = find_result_data(resultid=resultid, datatime_start=date_start, datatime_end=date_end, resolution_type='4KM') results_files = [row.address for row in results] results_files = set(results_files) session = Session() count = 0 for root, dirs, files in os.walk(source_dir): for file_name in files: if ext is not None: if '.' not in ext: ext = '.' + ext if os.path.splitext(file_name)[1].lower() != ext.lower(): continue try: src_file = os.path.join(root, file_name) datatime = FY4ASSI.get_date_time_orbit(src_file) if date_start is not None and date_end is not None: if not date_start <= datatime <= date_end: continue yyyymmdd = datatime.strftime("%Y%m%d") dst_file = os.path.join(ssi_dir, yyyymmdd, file_name) dst_file = dst_file.replace('4000M', '4KM') if not os.path.isfile(dst_file): dst_dir = os.path.dirname(dst_file) if not os.path.isdir(dst_dir): os.makedirs(dst_dir) os.symlink(src_file, dst_file) if dst_file in results_files: continue result_data = ResultData() result_data.resultid = resultid result_data.planid = planid result_data.address = dst_file result_data.datatime = datatime result_data.createtime = datetime.now() result_data.resolution_type = '4KM' result_data.area_type = 'Full_DISK' result_data.element = None session.add(result_data) count += 1 print('{} -----> {}'.format(src_file, dst_file)) if count >= 500: session.commit() count = 0 except Exception as why: print(why) os.remove(dst_file) session.commit() session.close()