def match_iss_avhrr(issObj,imagerGeoObj,imagerObj,ctype,cma,ctth,nwp,imagerAngObj,
cpp, nwp_segments):
import string
import os
retv = IssAvhrrTrackObject()
lonIss = issObj.longitude.ravel()
latIss = issObj.latitude.ravel()
timeIss = issObj.sec_1970.ravel()
from common import map_avhrr
cal, cap = map_avhrr(imagerGeoObj, lonIss.ravel(), latIss.ravel(),
radius_of_influence=RESOLUTION*0.7*1000.0) # larger than radius...
calnan = np.where(cal == NODATA, np.nan, cal)
import matplotlib.pyplot as plt
print lonIss.ravel(), latIss.ravel()
plt.plot(lonIss.ravel(), latIss.ravel(),'b.')
plt.plot(imagerGeoObj.longitude.ravel(), imagerGeoObj.latitude.ravel(),'r.')
plt.savefig('iss.png')
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
#check if it is within time limits:
if len(imagerGeoObj.time.shape)>1:
imager_time_vector = [imagerGeoObj.time[line,pixel] for line, pixel
in zip(cal,cap)]
imager_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector, np.nan)
else:
imager_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal], np.nan)
# Find all matching Iss pixels within +/- sec_timeThr from the AVHRR data
idx_match = elements_within_range(issObj.sec_1970, imager_lines_sec_1970, sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s."
% sec_timeThr)
retv.iss = calipso_track_from_matched(retv.iss, issObj, idx_match)
# Iss line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.iss.avhrr_linnum = cal_on_avhrr.astype('i')
retv.iss.avhrr_pixnum = cap_on_avhrr.astype('i')
# Imager time
if len(imagerGeoObj.time.shape)>1:
retv.avhrr.sec_1970= [imagerGeoObj.time[line,pixel] for line, pixel
in zip(cal_on_avhrr,cap_on_avhrr)]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.iss.sec_1970 - retv.avhrr.sec_1970
do_some_logging(retv, issObj)
logger.info("Generate the latitude,cloudtype tracks!")
retv = avhrr_track_from_matched(retv, imagerGeoObj, imagerObj, imagerAngObj,
nwp, ctth, ctype, cma, cal_on_avhrr, cap_on_avhrr,
cpp=cpp, nwp_segments=nwp_segments)
return retv
def match_cloudsat_avhrr(ctypefile, cloudsatObj, avhrrGeoObj, avhrrObj, ctype,
ctth, surft, avhrrAngObj, avhrrLwp, options):
import numpy
import time
import string
# import sys
# import inspect
import os
retv = CloudsatAvhrrTrackObject()
if RESOLUTION == 1:
lonCloudsat = cloudsatObj.longitude.ravel()
latCloudsat = cloudsatObj.latitude.ravel()
elif RESOLUTION == 5:
lonCloudsat = cloudsatObj.longitude[:, 1].ravel()
latCloudsat = cloudsatObj.latitude[:, 1].ravel()
timeCloudsat = cloudsatObj.sec1970.ravel()
ndim = lonCloudsat.shape[0]
# --------------------------------------------------------------------
#cal,cap = get_cloudsat_avhrr_linpix(avhrrGeoObj,ctypefile,lonCloudsat,latCloudsat,timeCloudsat, options)
# This function (match_calipso_avhrr) could use the MatchMapper object
# created in map_avhrr() to make things a lot simpler... See usage in
# amsr_avhrr_match.py
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
from common import map_avhrr
cal, cap = map_avhrr(imagerGeoObj,
lonCalipso.ravel(),
latCalipso.ravel(),
radius_of_influence=RESOLUTION * 0.7 *
1000.0) # somewhat larger than radius...
calnan = numpy.where(cal == NODATA, numpy.nan, cal)
if (~numpy.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
avhrr_lines_sec_1970 = numpy.where(cal != NODATA, avhrrGeoObj.time[cal],
numpy.nan)
# Find all matching Cloudsat pixels within +/- sec_timeThr from the AVHRR data
idx_match = elements_within_range(cloudsatObj.sec1970,
avhrr_lines_sec_1970, sec_timeThr)
# numpy.logical_and(cloudsatObj.sec1970 > avhrr_lines_sec_1970 - sec_timeThr,
# cloudsatObj.sec1970 < avhrr_lines_sec_1970 + sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." %
sec_timeThr)
duplicate_names(cloudsatObj)
#arnamecl = array name from cloudsatObj
for arnamecl, value in cloudsatObj.all_arrays.items():
if value != None:
if value.ndim == 0:
retv.cloudsat.all_arrays[arnamecl] = value.copy()
elif value.ndim == 1:
if value.size != 1:
retv.cloudsat.all_arrays[arnamecl] = value.copy()[
idx_match, :].astype('d')
elif value.ndim == 2:
temp = value.copy()[idx_match, :].astype('d')
if arnamecl == 'Radar_Reflectivity':
temp = numpy.where(numpy.less(temp, 0), -9.9, temp)
retv.cloudsat.all_arrays[arnamecl] = temp.transpose()
# Special because in 5km lon and lat is 2dim and shold just be 1dim
retv.cloudsat.longitude = numpy.repeat(lonCloudsat, idx_match).astype('d')
retv.cloudsat.latitude = numpy.repeat(latCloudsat, idx_match).astype('d')
# Cloudsat line,pixel inside AVHRR swath:
cal_on_avhrr = numpy.repeat(cal, idx_match)
cap_on_avhrr = numpy.repeat(cap, idx_match)
retv.cloudsat.avhrr_linnum = cal_on_avhrr.astype('i')
retv.cloudsat.avhrr_pixnum = cap_on_avhrr.astype('i')
N = retv.cloudsat.sec_1970.shape[0]
print "Cloudsat observation time of first cloudsat-avhrr match: ",\
time.gmtime(retv.cloudsat.sec_1970[0])
print "Cloudsat observation time of last cloudsat-avhrr match: ",\
time.gmtime(retv.cloudsat.sec_1970[N-1])
# Time
retv.avhrr.sec_1970 = avhrrGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.cloudsat.sec_1970 - retv.avhrr.sec_1970
min_diff = numpy.minimum.reduce(retv.diff_sec_1970)
max_diff = numpy.maximum.reduce(retv.diff_sec_1970)
print "Maximum and minimum time differences in sec (avhrr-cloudsat): ",\
numpy.maximum.reduce(retv.diff_sec_1970),numpy.minimum.reduce(retv.diff_sec_1970)
print "AVHRR observation time of first cloudsat-avhrr match: ",\
time.gmtime(retv.avhrr.sec_1970[0])
print "AVHRR observation time of last cloudsat-avhrr match: ",\
time.gmtime(retv.avhrr.sec_1970[N-1])
# Make the latitude and pps cloudtype on the cloudsat track:
# line and pixel arrays have equal dimensions
print "Generate all datatypes (lat,lon,cty,ctth,surft) on the cloudsat track!"
retv = avhrr_track_from_matched(retv, avhrrGeoObj, avhrrObj, avhrrAngObj, \
surft, ctth, ctype, cal_on_avhrr, cap_on_avhrr, avhrrLwp)
print "AVHRR-PPS Cloud Type,latitude: shapes = ",\
retv.avhrr.cloudtype.shape,retv.avhrr.latitude.shape
ll = []
for i in range(ndim):
#ll.append(("%7.3f %7.3f %d\n"%(lonCloudsat[i],latCloudsat[i],0)))
ll.append(("%7.3f %7.3f %d\n" %
(lonCloudsat[i], latCloudsat[i], idx_match[i])))
basename = os.path.basename(ctypefile).split(".h5")[0]
values = {"satellite": basename.split("_")[-8]}
values["year"] = str(basename.split("_")[-7][0:4])
values["month"] = str(basename.split("_")[-7][4:6])
values["basename"] = string.join(basename.split("_")[0:4], "_")
data_path = options['data_dir'].format(val_dir=_validation_results_dir,
satellite=values["satellite"],
resolution=str(RESOLUTION),
year=values["year"],
month=values["month"],
area=AREA)
if not os.path.exists(data_path):
print "Creating datadir: %s" % (data_path)
os.makedirs(data_path)
data_file = options['data_file'].format(resolution=str(RESOLUTION),
basename=values["basename"],
atrain_sat="cloudsat-GEOPROF",
track="track2")
filename = data_path + data_file
fd = open(filename, "w")
fd.writelines(ll)
fd.close()
ll = []
for i in range(N):
ll.append(("%7.3f %7.3f %d\n" %
(retv.cloudsat.longitude[i], retv.cloudsat.latitude[i], 0)))
data_file = options['data_file'].format(resolution=str(RESOLUTION),
basename=values["basename"],
atrain_sat="cloudsat-GEOPROF",
track="track_excl")
filename = data_path + data_file
fd = open(filename, "w")
fd.writelines(ll)
fd.close()
return retv, min_diff, max_diff
def match_calipso_avhrr(values,
calipsoObj, calipsoObjAerosol,
imagerGeoObj, imagerObj,
ctype, cma, ctth, cppCph, nwp_obj,
avhrrAngObj, nwp_segments, options, res=resolution):
import time
import string
retv = CalipsoAvhrrTrackObject()
dsec = time.mktime((1993,1,1,0,0,0,0,0,0)) - time.timezone # Convert from TAI time to UTC in seconds since 1970
if res == 1:
lonCalipso = calipsoObj.longitude.ravel()
latCalipso = calipsoObj.latitude.ravel()
timeCalipso_tai = calipsoObj.profile_time_tai[::,0].ravel()
timeCalipso = calipsoObj.profile_time_tai[::,0].ravel() + dsec
timeCalipso_utc = calipsoObj.profile_utc_time[::,0].ravel()
elevationCalipso = calipsoObj.dem_surface_elevation.ravel()
if res == 5:
# Use [:,1] Since 5km data has start, center, and end for each pixel
lonCalipso = calipsoObj.longitude[:,1].ravel()
latCalipso = calipsoObj.latitude[:,1].ravel()
timeCalipso_tai = calipsoObj.profile_time_tai[:,1].ravel()
timeCalipso = calipsoObj.profile_time_tai[:,1].ravel() + dsec
timeCalipso_utc = calipsoObj.profile_utc_time[:,1].ravel()
elevationCalipso = calipsoObj.dem_surface_elevation[::,2].ravel()
ndim = lonCalipso.shape[0]
# --------------------------------------------------------------------
#cal,cap = get_calipso_avhrr_linpix(imagerGeoObj,values,lonCalipso,latCalipso,timeCalipso, options)
# This function (match_calipso_avhrr) could use the MatchMapper object
# created in map_avhrr() to make things a lot simpler... See usage in
# amsr_avhrr_match.py
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
from common import map_avhrr
cal, cap = map_avhrr(imagerGeoObj, lonCalipso.ravel(), latCalipso.ravel(),
radius_of_influence=RESOLUTION*0.7*1000.0) # somewhat larger than radius...
print cal, cap
calnan = np.where(cal == NODATA, np.nan, cal)
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
if len(imagerGeoObj.time.shape)>1:
imager_time_vector = [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal,cap)]
avhrr_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector, np.nan)
else:
avhrr_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal], np.nan)
# avhrr_lines_sec_1970 = calnan * DSEC_PER_AVHRR_SCALINE + imagerGeoObj.sec1970_start
# Find all matching Calipso pixels within +/- sec_timeThr from the AVHRR data
# pdb.set_trace()
"""
import matplotlib.pyplot as plt
fig = plt.figure(figsize = (9,8))
ax = fig.add_subplot(111)
plt.plot(timeCalipso, 'b.')
plt.plot(avhrr_lines_sec_1970, 'g.')
plt.show()
fig.savefig("nina_test_fil.png")
"""
idx_match = elements_within_range(timeCalipso, avhrr_lines_sec_1970, sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." % sec_timeThr)
retv.calipso = calipso_track_from_matched(retv.calipso, calipsoObj, idx_match)
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
logger.info("cap_on_avhrr.shape: %s",cap_on_avhrr.shape)
lon_calipso = np.repeat(lonCalipso, idx_match)
lat_calipso = np.repeat(latCalipso, idx_match)
# Calipso line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
logger.info("Start and end times: %s %s",
time.gmtime(timeCalipso[0]),
time.gmtime(timeCalipso[ndim-1]))
retv.calipso.sec_1970 = np.repeat(timeCalipso,idx_match)
retv.calipso.latitude = np.repeat(latCalipso,idx_match)
retv.calipso.longitude = np.repeat(lonCalipso,idx_match)
retv.calipso.profile_time_tai = np.repeat(timeCalipso_tai,idx_match)
# Elevation is given in km's. Convert to meters:
retv.calipso.elevation = np.repeat(elevationCalipso.ravel()*1000.0,
idx_match.ravel()).astype('d')
# Time
if len(imagerGeoObj.time.shape)>1:
retv.avhrr.sec_1970= [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal_on_avhrr,cap_on_avhrr)]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970
min_diff = np.minimum.reduce(retv.diff_sec_1970)
max_diff = np.maximum.reduce(retv.diff_sec_1970)
logger.info("Maximum and minimum time differences in sec (avhrr-calipso): %d %d",
np.maximum.reduce(retv.diff_sec_1970),np.minimum.reduce(retv.diff_sec_1970))
logger.info("AVHRR observation time of first calipso-avhrr match: %s",
time.gmtime(retv.avhrr.sec_1970[0]))
logger.info("AVHRR observation time of last calipso-avhrr match: %s",
time.gmtime(retv.avhrr.sec_1970[-1]))
# Make the latitude and pps cloudtype on the calipso track:
# line and pixel arrays have equal dimensions
logger.info("Generate the latitude,cloudtype tracks!")
# -------------------------------------------------------------------------
# Pick out the data from the track from AVHRR
from extract_imager_along_track import avhrr_track_from_matched
retv = avhrr_track_from_matched(retv, imagerGeoObj, imagerObj, avhrrAngObj,
nwp_obj, ctth, ctype, cma, cal_on_avhrr,
cap_on_avhrr, avhrrCph=cppCph,
nwp_segments=nwp_segments)
if calipsoObjAerosol is not None:
retv.calipso_aerosol = calipso_track_from_matched(retv.calipso_aerosol, calipsoObjAerosol, idx_match)
# -------------------------------------------------------------------------
logger.info("AVHRR-PPS Cloud Type,latitude: shapes = %s %s",
retv.avhrr.cloudtype.shape,retv.avhrr.latitude.shape)
ll = []
for i in range(ndim):
#ll.append(("%7.3f %7.3f %d\n"%(lonCalipso[i],latCalipso[i],0)))
ll.append(("%7.3f %7.3f %d\n"%(lonCalipso[i],latCalipso[i],idx_match[i])))
max_cloud_top_calipso = np.maximum.reduce(retv.calipso.layer_top_altitude.ravel())
logger.info("max_cloud_top_calipso: %2.1f",max_cloud_top_calipso)
return retv,min_diff,max_diff
def match_calipso_avhrr(values,
caObj,
caObjAerosol,
imagerGeoObj,
imagerObj,
ctype,
cma,
ctth,
cpp,
nwp_obj,
avhrrAngObj,
nwp_segments,
options,
res=resolution):
import string
from common import map_avhrr
retv = CalipsoAvhrrTrackObject()
lonCalipso = caObj.longitude.ravel()
latCalipso = caObj.latitude.ravel()
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
cal, cap = map_avhrr(imagerGeoObj,
lonCalipso.ravel(),
latCalipso.ravel(),
radius_of_influence=RESOLUTION * 0.7 *
1000.0) # somewhat larger than radius...
#warn if no matches
calnan = np.where(cal == NODATA, np.nan, cal)
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
#check if it is within time limits:
if len(imagerGeoObj.time.shape) > 1:
imager_time_vector = [
imagerGeoObj.time[line, pixel] for line, pixel in zip(cal, cap)
]
imager_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector,
np.nan)
else:
imager_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal],
np.nan)
idx_match = elements_within_range(caObj.sec_1970, imager_lines_sec_1970,
sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." %
sec_timeThr)
retv.calipso = calipso_track_from_matched(retv.calipso, caObj, idx_match)
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
#logger.info("calipso matched with avhrr shape: %s",cap_on_avhrr.shape)
# Calipso line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
# Imager time
if len(imagerGeoObj.time.shape) > 1:
retv.avhrr.sec_1970 = [
imagerGeoObj.time[line, pixel]
for line, pixel in zip(cal_on_avhrr, cap_on_avhrr)
]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970
do_some_logging(retv, caObj)
logger.info("Generate the latitude,cloudtype tracks!")
from extract_imager_along_track import avhrr_track_from_matched
retv = avhrr_track_from_matched(retv,
imagerGeoObj,
imagerObj,
avhrrAngObj,
nwp_obj,
ctth,
ctype,
cma,
cal_on_avhrr,
cap_on_avhrr,
cpp=cpp,
nwp_segments=nwp_segments)
if caObjAerosol is not None:
retv.calipso_aerosol = calipso_track_from_matched(
retv.calipso_aerosol, caObjAerosol, idx_match)
max_cloud_top_calipso = np.maximum.reduce(
retv.calipso.layer_top_altitude.ravel())
logger.info("max_cloud_top_calipso: %2.1f", max_cloud_top_calipso)
return retv
def match_cloudsat_avhrr(cloudsatObj, imagerGeoObj, imagerObj, ctype, cma,
ctth, nwp, imagerAngObj, cpp, nwp_segments):
import string
import os
retv = CloudsatAvhrrTrackObject()
lonCloudsat = cloudsatObj.longitude.ravel()
latCloudsat = cloudsatObj.latitude.ravel()
timeCloudsat = cloudsatObj.sec_1970.ravel()
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
from common import map_avhrr
cal, cap = map_avhrr(imagerGeoObj,
lonCloudsat.ravel(),
latCloudsat.ravel(),
radius_of_influence=RESOLUTION * 0.7 *
1000.0) # somewhat larger than radius...
calnan = np.where(cal == NODATA, np.nan, cal)
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
#check if it is within time limits:
if len(imagerGeoObj.time.shape) > 1:
imager_time_vector = [
imagerGeoObj.time[line, pixel] for line, pixel in zip(cal, cap)
]
imager_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector,
np.nan)
else:
imager_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal],
np.nan)
# Find all matching Cloudsat pixels within +/- sec_timeThr from the AVHRR data
idx_match = elements_within_range(cloudsatObj.sec_1970,
imager_lines_sec_1970, sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." %
sec_timeThr)
retv.cloudsat = calipso_track_from_matched(retv.cloudsat, cloudsatObj,
idx_match)
# Cloudsat line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.cloudsat.avhrr_linnum = cal_on_avhrr.astype('i')
retv.cloudsat.avhrr_pixnum = cap_on_avhrr.astype('i')
# Imager time
if len(imagerGeoObj.time.shape) > 1:
retv.avhrr.sec_1970 = [
imagerGeoObj.time[line, pixel]
for line, pixel in zip(cal_on_avhrr, cap_on_avhrr)
]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.cloudsat.sec_1970 - retv.avhrr.sec_1970
do_some_logging(retv, cloudsatObj)
logger.info("Generate the latitude,cloudtype tracks!")
retv = avhrr_track_from_matched(retv,
imagerGeoObj,
imagerObj,
imagerAngObj,
nwp,
ctth,
ctype,
cma,
cal_on_avhrr,
cap_on_avhrr,
cpp=cpp,
nwp_segments=nwp_segments)
return retv