def product_fy4a_1km_disk_full_data_orbit(date_start=None,
date_end=None,
thread=THREAD,
**kwargs):
"""
绘制原始4KM数据的图像
3个产品，每个产品2张图像，共6张图像
:param date_start: 开始日期 datetime
:param date_end: 结束日期 datetime
:param thread:
:return:
"""
resultid_in = 'FY4A_AGRI_L2_SSI_Orbit'
resultid_out = 'FY4A_AGRI_L2_SSI_Orbit'
frequency = 'Orbit'
resolution_type_in = '4KMCorrect'
resolution_type = '1KM'
out_dir = os.path.join(
DATA_ROOT_DIR, 'SSIData/FY4A/SSI_{resolution_type}/Full/{frequency}')
out_dir = out_dir.format(resolution_type=resolution_type,
frequency=frequency)
planid = 1
results = find_result_data(resultid=resultid_in,
datatime_start=date_start,
datatime_end=date_end,
resolution_type=resolution_type_in)
in_files = [row.address for row in results]
in_files.sort()
in_files_length = len(in_files)
print('找到的文件总数:{}'.format(in_files_length))
print('开始生产')
p = Pool(thread)
for in_file in in_files[:]:
in_file_name = os.path.basename(in_file)
datatime = FY4ASSI.get_date_time_orbit(in_file)
if datatime.minute != 0:
continue
date_time = datatime.strftime('%Y%m%d%H%M%S')
out_file_name = in_file_name.replace(resolution_type_in,
resolution_type)
out_file = os.path.join(out_dir, date_time[:8], out_file_name)
if DEBUG or thread == 1:
fy4a_ssi_4km_to_1km(in_file, out_file, resultid_out, planid,
datatime, resolution_type)
else:
p.apply_async(fy4a_ssi_4km_to_1km,
args=(in_file, out_file, resultid_out, planid,
datatime, resolution_type))
p.close()
p.join()
print('完成全部的任务:{}'.format(sys._getframe().f_code.co_name))
def product_cloud_image(date_start=None,
date_end=None,
frequency=None,
thread=THREAD,
resolution_type=None,
sat_sensor=None,
**kwargs):
sat_sensor = sat_sensor.upper()
sat, sensor = sat_sensor.split('_')
if frequency == 'Orbit':
resultid_data_in = '{}_L1_FDI_Orbit'.format(sat_sensor)
if sat == 'FY4A':
vmin, vmax, _ = COLORBAR_RANGE_ORBIT_FY4A
else:
print('此时间分辨率={}不支持卫星={}'.format(frequency, sat_sensor))
return
planid = 1
get_date_time = FY4ASSI.get_date_time_orbit
else:
raise ValueError('不支持的类型：{}'.format(frequency))
resultid_image = resultid_data_in + '_IMG'
try:
results = find_result_data(resultid=resultid_data_in,
datatime_start=date_start,
datatime_end=date_end,
resolution_type=resolution_type)
except Exception as why:
print(why)
return
in_files = [row.address for row in results]
in_files.sort()
in_files_length = len(in_files)
print('找到的文件总数:{}'.format(in_files_length))
print('开始绘图')
p = Pool(thread)
for in_file in in_files[:]:
datatime = get_date_time(in_file)
if DEBUG or thread == 1:
plot_cloud_full(in_file, resultid_image, planid, datatime,
resolution_type)
else:
p.apply_async(plot_cloud_full,
args=(in_file, resultid_image, planid, datatime,
resolution_type))
p.close()
p.join()
print('完成全部的任务:{}'.format(sys._getframe().f_code.co_name))
def product_point_data(date_start=None,
date_end=None,
lon=None,
lat=None,
point_file=None,
resolution_type=None,
resultid=None,
element=None,
idx=None,
ck=None,
sat_sensor=None,
**kwargs):
sat_sensor = sat_sensor.upper()
sat, sensor = sat_sensor.split('_')
date_s = datetime.strptime(date_start, '%Y%m%d%H%M%S')
date_e = datetime.strptime(date_end, '%Y%m%d%H%M%S')
date_start_beijing = (date_s +
relativedelta(hours=8)).strftime("%Y%m%d%H%M%S")
date_end_beijing = (date_e +
relativedelta(hours=8)).strftime("%Y%m%d%H%M%S")
out_dir = os.path.join(DATA_ROOT_DIR, 'TmpData', '{}'.format(sat))
outname = '{sat}-_{sensor}--_{lon:07.3f}N_DISK_{lat:07.3f}E_L2-_SSI-_MULT_NOM_' \
'{date_start}_{date_end}_{resolution_type}_V0001_{site}.TXT'
if 'fy3d' in sat.lower():
loader = FY4ASSI
elif 'fy4a' in sat.lower():
loader = FY3DSSI
else:
raise ValueError('不支持的卫星：{}'.format(sat))
if 'Orbit' in resultid:
get_datetime = loader.get_date_time_orbit
elif 'Daily' in resultid:
get_datetime = loader.get_date_time_daily
elif 'Monthly' in resultid:
get_datetime = loader.get_date_time_monthly
elif 'Yearly' in resultid:
get_datetime = loader.get_date_time_yearly
else:
print('不支持的时间频率:{}'.format(resultid))
return
results = find_result_data(resultid=resultid,
datatime_start=date_s,
datatime_end=date_e,
resolution_type=resolution_type)
full_files = [row.address for row in results]
full_files.sort()
in_files_length = len(full_files)
print('找到的文件总数:{}'.format(in_files_length))
if in_files_length <= 0:
return
if point_file is None: # 单点数据
lon = float(lon)
lat = float(lat)
pre_dist = 0.08
index = get_point_index(lon, lat, idx, ck, pre_dist)
print('index', index)
if index is None:
print('没有找到最近点:lon {} lat:{}'.format(lon, lat))
return
datas = []
dates = []
values = []
count = 0
s = datetime.now()
thread_lock = threading.Lock()
for full_file in full_files:
if not os.path.isfile(full_file):
print('文件不存在: {}'.format(full_file))
count += 1
continue
_get_point_data(full_file, element, index, get_datetime, resultid,
dates, datas, values, thread_lock)
# thread_lock = threading.Lock()
# ts = []
# for full_file in full_files:
# if not os.path.isfile(full_file):
# print('文件不存在: {}'.format(full_file))
# count += 1
# continue
# t = threading.Thread(target=_get_point_data, args=(full_file, element, index, get_datetime, resultid,
# dates, datas, values, thread_lock))
# t.start()
# ts.append(t)
# for t in ts:
# t.join()
print('获取{}数据消耗时间:{}'.format(in_files_length, datetime.now() - s))
if len(datas) <= 0:
print('没有找到有效的点数据')
return
if element is not None: # 返回单点数据
dates_new = []
values_new = []
for date, value in sorted(zip(dates, values), key=lambda x: x[0]):
dates_new.append(date)
values_new.append(float(value[element]))
if len(values_new) > 0:
return {
'date': dates_new,
'value': values_new,
'values': values
}
else:
print('没有找到有效的点数据')
return
else: # 返回单点的TXT文件
out_file = os.path.join(
out_dir,
outname.format(sat=sat,
sensor=sensor,
lon=lon,
lat=lat,
date_start=date_start_beijing,
date_end=date_end_beijing,
resolution_type=resolution_type,
site=''))
if len(datas) > 0:
with open(out_file, 'w') as fp:
header = "Date\tItol\tIb\tId\tG0\tGt\tDNI\n"
fp.write(header)
datas.sort()
fp.writelines(datas)
return out_file
else:
print('没有找到有效的点数据')
return
else: # 多点数据
point_infos = []
indexs = []
with open(point_file, 'r') as fp:
fp.readline()
for row in fp.readlines():
point_name, lon, lat = row.strip().split()
lon = float(lon)
lat = float(lat)
pre_dist = 0.08
index = get_point_index(lon, lat, idx, ck, pre_dist)
if index is None:
print('没有找到最近点:lon {} lat:{}'.format(lon, lat))
continue
else:
point_infos.append((point_name, lon, lat))
print(type(index[0]))
indexs.append([int(index[0]), int(index[1])])
for info in point_infos:
print(info)
dates = []
values = []
count = 0
s = datetime.now()
thread_lock = threading.Lock()
for full_file in full_files:
if not os.path.isfile(full_file):
print('文件不存在: {}'.format(full_file))
count += 1
continue
_get_multi_point_data(full_file, indexs, get_datetime, resultid,
dates, values, thread_lock)
print('获取{}数据消耗时间:{}'.format(in_files_length, datetime.now() - s))
print(values)
out_files = []
for i, info in enumerate(point_infos): # i是站点的索引值
point_name, lon, lat = info
print('站点名: {}'.format(point_name))
datas = []
for j, date in enumerate(dates): # j 是日期的索引值
print('日期：{}'.format(date))
datas_tmp = values[j]
data_format = {'date': date}
for element_ in datas_tmp:
data_ = datas_tmp[element_][i]
data_format[element_] = data_
# if data_ == 0:
# data_format[element_] = data_
# else:
# data_format[element_] = np.nan
data_str = '{date}\t{Itol:0.4f}\t{Ib:0.4f}\t{Id:0.4f}\t{G0:0.4f}\t{Gt:0.4f}\t{DNI:0.4f}\n'.format(
**data_format)
datas.append(data_str)
out_file = os.path.join(
out_dir,
outname.format(sat=sat,
sensor=sensor,
lon=lon,
lat=lat,
date_start=date_start_beijing,
date_end=date_end_beijing,
resolution_type=resolution_type,
site=point_name))
with open(out_file, 'w') as fp:
header = "Date\tItol\tIb\tId\tG0\tGt\tDNI\n"
fp.write(header)
datas.sort()
fp.writelines(datas)
out_files.append(out_file)
return out_files
def product_area_data(date_start=None,
date_end=None,
thread=THREAD,
left_up_lon=None,
left_up_lat=None,
right_down_lon=None,
right_down_lat=None,
resolution_type=None,
resultid=None,
**kwargs):
"""
生成原始数据的中国区时次数据
3个产品，每个产品1张图像，共3张图像
:param date_start: 开始日期 datetime
:param date_end: 结束日期 datetime
:param thread:
:param left_up_lon:
:param left_up_lat:
:param right_down_lon:
:param right_down_lat:
:param resolution_type:
:param resultid:
:return:
"""
frequency = resultid.split('_')[-1]
if isinstance(date_start, str):
date_start = datetime.strptime(date_start, '%Y%m%d%H%M%S')
date_end = datetime.strptime(date_end, '%Y%m%d%H%M%S')
results = find_result_data(resultid=resultid,
datatime_start=date_start,
datatime_end=date_end,
resolution_type=resolution_type)
in_files = [row.address for row in results]
in_files.sort()
in_files_length = len(in_files)
print('找到的文件总数:{}'.format(in_files_length))
sat, sensor = resultid.split('_')[:2]
out_dir = os.path.join(
DATA_ROOT_DIR, 'TmpData',
'{sat}/SSI_{resolution_type}/Area/{frequency}'.format(
sat=sat, resolution_type=resolution_type, frequency=frequency))
out_dir = os.path.join(
out_dir,
'{:08.4f}_{:08.4f}_{:08.4f}_{:08.4f}'.format(left_up_lon, left_up_lat,
right_down_lon,
right_down_lat))
print('开始生成区域数据')
print(left_up_lon, left_up_lat, right_down_lon, right_down_lat)
out_files = []
if DEBUG or thread == 1:
for in_file in in_files[:]:
filename = os.path.basename(in_file)
out_file = os.path.join(out_dir, filename)
r = area(in_file, out_file, left_up_lon, left_up_lat,
right_down_lon, right_down_lat, resolution_type, resultid)
if r is not None:
out_files.append(r)
else:
p = Pool(thread)
for in_file in in_files[:]:
filename = os.path.basename(in_file)
out_file = os.path.join(out_dir, filename)
out_files.append(out_file)
p.apply_async(area,
args=(in_file, out_file, left_up_lon, left_up_lat,
right_down_lon, right_down_lat,
resolution_type, resultid))
p.close()
p.join()
if len(out_files) > 0:
return out_files
else:
return
def fy4a_save_4km_orbit_data_in_database(date_start=None,
date_end=None,
**kwargs):
print(date_start)
print(date_end)
# source_dir = os.path.join('/FY4/FY4A/AGRI/L2/SSI/DISK/NOM')
source_dir = os.path.join(
'/home/gfssi/GFData/SourceData/FY4A/SSI_4KM/Full/Orbit')
ssi_dir = os.path.join(DATA_ROOT_DIR, 'SSIData', 'FY4A', 'SSI_4KM', 'Full',
'Orbit')
ext = '.NC'
resultid = 'FY4A_AGRI_L2_SSI_Orbit'
planid = 1
results = find_result_data(resultid=resultid,
datatime_start=date_start,
datatime_end=date_end,
resolution_type='4KM')
results_files = [row.address for row in results]
results_files = set(results_files)
session = Session()
count = 0
for root, dirs, files in os.walk(source_dir):
for file_name in files:
if ext is not None:
if '.' not in ext:
ext = '.' + ext
if os.path.splitext(file_name)[1].lower() != ext.lower():
continue
try:
src_file = os.path.join(root, file_name)
datatime = FY4ASSI.get_date_time_orbit(src_file)
if date_start is not None and date_end is not None:
if not date_start <= datatime <= date_end:
continue
yyyymmdd = datatime.strftime("%Y%m%d")
dst_file = os.path.join(ssi_dir, yyyymmdd, file_name)
dst_file = dst_file.replace('4000M', '4KM')
if not os.path.isfile(dst_file):
dst_dir = os.path.dirname(dst_file)
if not os.path.isdir(dst_dir):
os.makedirs(dst_dir)
os.symlink(src_file, dst_file)
if dst_file in results_files:
continue
result_data = ResultData()
result_data.resultid = resultid
result_data.planid = planid
result_data.address = dst_file
result_data.datatime = datatime
result_data.createtime = datetime.now()
result_data.resolution_type = '4KM'
result_data.area_type = 'Full_DISK'
result_data.element = None
session.add(result_data)
count += 1
print('{} -----> {}'.format(src_file, dst_file))
if count >= 500:
session.commit()
count = 0
except Exception as why:
print(why)
os.remove(dst_file)
session.commit()
session.close()
def product_combine_data(date_start=None,
date_end=None,
frequency=None,
thread=THREAD,
resolution_type=None,
sat_sensor=None):
sat_sensor = sat_sensor.upper()
sat, sensor = sat_sensor.split('_')
out_dir = os.path.join(
DATA_ROOT_DIR, 'SSIData/{sat}/SSI_{resolution_type}/Full/{frequency}')
out_dir = out_dir.format(sat=sat,
resolution_type=resolution_type,
frequency=frequency)
if sat == 'FY4A':
out_name = 'FY4A-_AGRI--_N_DISK_1047E_L3-_SSI-_MULT_NOM_{date}_{resolution_type}_V0001.NC'
elif sat == 'FY3D':
out_name = 'FY3D-_MERSI--_N_DISK_E_L3-_SSI-_MULT_NOM_{date}_{resolution_type}_V0001.NC'
else:
print('不支持的卫星:sat_sensor={}'.format(sat_sensor))
return
if frequency == 'Daily':
resultid_data_in = '{}_L2_SSI_Orbit'.format(sat_sensor)
strf_name = '%Y%m%d'
strf_dir = '%Y%m'
date_relativedelta = relativedelta(days=1)
daily = True
planid = 1
get_date_time = FY4ASSI.get_date_time_daily
elif frequency == 'Monthly':
resultid_data_in = '{}_L3_SSI_Daily'.format(sat_sensor)
out_dir = out_dir.format(resolution_type=resolution_type,
frequency='Monthly')
strf_name = '%Y%m'
strf_dir = '%Y'
date_relativedelta = relativedelta(months=1)
daily = False
planid = 1
get_date_time = FY4ASSI.get_date_time_monthly
elif frequency == 'Yearly':
resultid_data_in = '{}_L3_SSI_Monthly'.format(sat_sensor)
out_dir = out_dir.format(resolution_type=resolution_type,
frequency='Yearly')
strf_name = '%Y'
strf_dir = None
date_relativedelta = relativedelta(years=1)
daily = False
planid = 1
get_date_time = FY4ASSI.get_date_time_yearly
else:
raise ValueError('不支持的类型：{}'.format(frequency))
resultid_combine = '{sat_sensor}_L3_SSI_{frequency}'.format(
sat_sensor=sat_sensor, frequency=frequency)
files_combine = []
p = Pool(thread)
date_tmp = date_start
while date_tmp <= date_end:
time.sleep(0.1)
date_name = date_tmp.strftime(strf_name)
if strf_dir is not None:
date_dir = date_tmp.strftime(strf_dir)
else:
date_dir = ''
date_s = datetime.strptime(date_name, strf_name)
date_e = date_s + date_relativedelta - relativedelta(seconds=1)
try:
results = find_result_data(resultid=resultid_data_in,
datatime_start=date_s,
datatime_end=date_e,
resolution_type=resolution_type)
except Exception as why:
print(why)
continue
in_files = [row.address for row in results]
in_files.sort()
in_files_length = len(in_files)
print('找到的文件总数:{}'.format(in_files_length))
out_name_tmp = out_name.format(date=date_name,
resolution_type=resolution_type)
out_file = os.path.join(out_dir, date_dir, out_name_tmp)
datatime = get_date_time(out_file)
if DEBUG or thread == 1:
combine_full(in_files, out_file, daily, resultid_combine, planid,
datatime, resolution_type)
else:
p.apply_async(combine_full,
args=(in_files, out_file, daily, resultid_combine,
planid, datatime, resolution_type))
files_combine.append(out_file)
date_tmp += date_relativedelta
p.close()
p.join()
print('完成全部的任务:{}'.format(sys._getframe().f_code.co_name))