def load(satscene, calibrate=True, area_extent=None, **kwargs): """Load MSG SEVIRI data from hdf5 format. """ # Read config file content conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) values = {"orbit": satscene.orbit, "satname": satscene.satname, "number": satscene.number, "instrument": satscene.instrument_name, "satellite": satscene.fullname } LOG.info("assume seviri-level4") print "... assume seviri-level4" satscene.add_to_history("hdf5 data read by mpop/msg_seviri_hdf.py") if "reader_level" in kwargs.keys(): reader_level = kwargs["reader_level"] else: reader_level = "seviri-level4" if "RSS" in kwargs.keys(): if kwargs["RSS"]: dt_end = 4 else: dt_end = 12 else: from my_msg_module import check_RSS RSS = check_RSS(satscene.sat_nr(), satscene.time_slot) if RSS == None: print "*** Error in mpop/satin/msg_seviri_hdf.py" print " satellite MSG", satscene.sat_nr() ," is not active yet" quit() else: if RSS: dt_end = 4 else: dt_end = 12 print "... hdf file name is specified by observation end time" print " assume ", dt_end, " min between start and end time of observation" # end of scan time 4 min after start end_time = satscene.time_slot + datetime.timedelta(minutes=dt_end) filename = os.path.join( end_time.strftime(conf.get(reader_level, "dir", raw=True)), end_time.strftime(conf.get(reader_level, "filename", raw=True)) % values ) print "... search for file: ", filename filenames=glob(str(filename)) if len(filenames) == 0: print "*** Error, no file found" return # just return without exit the program elif len(filenames) > 1: print "*** Warning, more than 1 datafile found: ", filenames filename = filenames[0] print("... read data from %s" % str(filename)) # read data from hdf5 file data_folder='U-MARF/MSG/Level1.5/' # Load data from hdf file with h5py.File(filename,'r') as hf: subset_info=hf.get(data_folder+'METADATA/SUBSET') for i in range(subset_info.len()): #print subset_info[i]['EntryName'], subset_info[i]['Value'] if subset_info[i]['EntryName'] == "VIS_IRSouthLineSelectedRectangle": VIS_IRSouthLine = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "VIS_IRNorthLineSelectedRectangle": VIS_IRNorthLine = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "VIS_IREastColumnSelectedRectangle": VIS_IREastColumn = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "VIS_IRWestColumnSelectedRectangle": VIS_IRWestColumn = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "HRVLowerNorthLineSelectedRectangle": HRVLowerNorthLine = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "HRVLowerSouthLineSelectedRectangle": HRVLowerSouthLine = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "HRVLowerEastColumnSelectedRectangle": HRVLowerEastColumn = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "HRVLowerWestColumnSelectedRectangle": HRVLowerWestColumn = int(subset_info[i]['Value']) if subset_info[i]['EntryName'] == "HRVUpperSouthLineSelectedRectangle": HRVUpperSouthLine = int(subset_info[i]['Value']) # 0 if subset_info[i]['EntryName'] == "HRVUpperNorthLineSelectedRectangle": HRVUpperNorthLine = int(subset_info[i]['Value']) # 0 if subset_info[i]['EntryName'] == "HRVUpperEastColumnSelectedRectangle": HRVUpperEastColumn = int(subset_info[i]['Value']) # 0 if subset_info[i]['EntryName'] == "HRVUpperWestColumnSelectedRectangle": HRVUpperWestColumn = int(subset_info[i]['Value']) # 0 sat_status=hf.get(data_folder+'METADATA/HEADER/SatelliteStatus/SatelliteStatus_DESCR') for i in range(subset_info.len()): if sat_status[i]['EntryName']=="SatelliteDefinition-NominalLongitude": sat_lon = sat_status[i]['Value'] break #print 'VIS_IRSouthLine', VIS_IRSouthLine #print 'VIS_IRNorthLine', VIS_IRNorthLine #print 'VIS_IREastColumn', VIS_IREastColumn #print 'VIS_IRWestColumn', VIS_IRWestColumn #print 'sat_longitude', sat_lon, type(sat_lon), 'GEOS<'+'{:+06.1f}'.format(sat_lon)+'>' if 1 == 0: # works only if all pixels are on the disk from msg_pixcoord2area import msg_pixcoord2area print "VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn: ", VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn area_def = msg_pixcoord2area ( VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn, "vis", sat_lon ) else: # works also for pixels outside of the disk pname = 'GEOS<'+'{:+06.1f}'.format(sat_lon)+'>' # "GEOS<+009.5>" proj = {'proj': 'geos', 'a': '6378169.0', 'b': '6356583.8', 'h': '35785831.0', 'lon_0': str(sat_lon)} aex=(-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612) # define full disk projection from pyresample.geometry import AreaDefinition full_disk_def = AreaDefinition('full_disk', 'full_disk', pname, proj, 3712, 3712, aex ) # define name and calculate area for sub-demain area_name= 'MSG_'+'{:04d}'.format(VIS_IRNorthLine)+'_'+'{:04d}'.format(VIS_IRWestColumn)+'_'+'{:04d}'.format(VIS_IRSouthLine)+'_'+'{:04d}'.format(VIS_IREastColumn) aex = full_disk_def.get_area_extent_for_subset(3712-VIS_IRSouthLine,3712-VIS_IRWestColumn,3712-VIS_IRNorthLine,3712-VIS_IREastColumn) area_def = AreaDefinition(area_name, area_name, pname, proj, (VIS_IRWestColumn-VIS_IREastColumn)+1, (VIS_IRNorthLine-VIS_IRSouthLine)+1, aex ) #print area_def #print "REGION:", area_def.area_id, "{" #print "\tNAME:\t", area_def.name #print "\tPCS_ID:\t", area_def.proj_id #print ("\tPCS_DEF:\tproj="+area_def.proj_dict['proj']+", lon_0=" + area_def.proj_dict['lon_0'] + ", a="+area_def.proj_dict['a']+", b="+area_def.proj_dict['b']+", h="+area_def.proj_dict['h']) #print "\tXSIZE:\t", area_def.x_size #print "\tYSIZE:\t", area_def.y_size #print "\tAREA_EXTENT:\t", area_def.area_extent #print "};" # copy area to satscene satscene.area = area_def # write information used by mipp.xrit.MSG._Calibrator in a fake header file hdr = dict() # satellite ID number hdr["SatelliteDefinition"] = dict() hdr["SatelliteDefinition"]["SatelliteId"] = SatelliteIds[str(satscene.sat_nr())] # processing hdr["Level 1_5 ImageProduction"] = dict() hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"] = np_array([2,2,2,2,2,2,2,2,2,2,2,2], int) # calibration factors Level15ImageCalibration = hf.get(data_folder+'METADATA/HEADER/RadiometricProcessing/Level15ImageCalibration_ARRAY') hdr["Level1_5ImageCalibration"] = dict() for chn_name in channel_numbers.keys(): chn_nb = channel_numbers[chn_name]-1 hdr["Level1_5ImageCalibration"][chn_nb] = dict() #print chn_name, chn_nb, Level15ImageCalibration[chn_nb]['Cal_Slope'], Level15ImageCalibration[chn_nb]['Cal_Offset'] hdr["Level1_5ImageCalibration"][chn_nb]['Cal_Slope'] = Level15ImageCalibration[chn_nb]['Cal_Slope'] hdr["Level1_5ImageCalibration"][chn_nb]['Cal_Offset'] = Level15ImageCalibration[chn_nb]['Cal_Offset'] # loop over channels to load for chn_name in satscene.channels_to_load: dataset_name = data_folder+'DATA/'+dict_channel[chn_name]+'/IMAGE_DATA' if dataset_name in hf: data_tmp = hf.get(data_folder+'DATA/'+dict_channel[chn_name]+'/IMAGE_DATA') LOG.info('hdr["SatelliteDefinition"]["SatelliteId"]: '+str(hdr["SatelliteDefinition"]["SatelliteId"])) #LOG.info('hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"]', hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"]) chn_nb = channel_numbers[chn_name]-1 LOG.info('hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Slope"]: '+str(hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Slope"])) LOG.info('hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Offset"]: '+str(hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Offset"])) if calibrate: #Calibrator = _Calibrator(hdr, chn_name) bits_per_pixel = 10 ### !!! I have no idea if this is correct !!! Calibrator = _Calibrator(hdr, chn_name, bits_per_pixel) ## changed call in mipp/xrit/MSG.py data, calibration_unit = Calibrator (data_tmp, calibrate=1) else: data = data_tmp calibration_unit = "counts" LOG.info(chn_name+ " min/max: "+str(data.min())+","+str(data.max())+" "+calibration_unit ) satscene[chn_name] = ma.asarray(data) satscene[chn_name].info['units'] = calibration_unit satscene[chn_name].info['satname'] = satscene.satname satscene[chn_name].info['satnumber'] = satscene.number satscene[chn_name].info['instrument_name'] = satscene.instrument_name satscene[chn_name].info['time'] = satscene.time_slot satscene[chn_name].info['is_calibrated'] = True else: print "*** Warning, no data for channel "+ chn_name+ " in file "+ filename data = np_nan calibration_unit = "" LOG.info("*** Warning, no data for channel "+ chn_name+" in file "+filename)
def _get_metadata(hdr, ftr, channel_name): md = Metadata() md.calibrate = _Calibrator(hdr, channel_name) if channel_name == "HRV": md.pixel_size = (1000.134348869, 1000.134348869) else: md.pixel_size = (3000.403165817, 3000.403165817) md.sublon = hdr["ProjectionDescription"]["LongitudeOfSSP"] md.channel = channel_name if md.channel == "HRV": md.image_size = np.array((hdr["ReferenceGridHRV"]["NumberOfLines"], hdr["ReferenceGridHRV"]["NumberOfColumns"])) else: md.image_size = np.array((hdr["ReferenceGridVIS_IR"]["NumberOfLines"], hdr["ReferenceGridVIS_IR"]["NumberOfColumns"])) #md.satname = im.platform.lower() md.product_type = 'full disc' md.region_name = 'full disc' if md.channel == "HRV": md.first_pixel = hdr["ReferenceGridHRV"]["GridOrigin"] ns_, ew_ = md.first_pixel.split() del ns_ md.boundaries = np.array([[ ftr["LowerSouthLineActual"], ftr["LowerNorthLineActual"], ftr["LowerEastColumnActual"], ftr["LowerWestColumnActual"]], [ftr["UpperSouthLineActual"], ftr["UpperNorthLineActual"], ftr["UpperEastColumnActual"], ftr["UpperWestColumnActual"]]]) hcoff = 1856 * 3 hloff = 1856 * 3 md.coff = (ftr["Lower"+ew_.capitalize()+"ColumnActual"] + hcoff - 1) md.loff = hloff else: md.first_pixel = hdr["ReferenceGridVIS_IR"]["GridOrigin"] ns_, ew_ = md.first_pixel.split() md.boundaries = np.array([[ ftr["SouthernLineActual"], ftr["NorthernLineActual"], ftr["EasternColumnActual"], ftr["WesternColumnActual"]]]) lcoff = 1856 lloff = 1856 md.coff = lcoff md.loff = lloff md.no_data_value = no_data_value md.line_offset = 0 #md.time_stamp = im.time_stamp #md.production_time = im.production_time md.calibration_unit = 'counts' return md
def load(satscene, calibrate=True, area_extent=None, **kwargs): """Load MSG SEVIRI data from hdf5 format. """ # Read config file content conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg")) values = { "orbit": satscene.orbit, "satname": satscene.satname, "number": satscene.number, "instrument": satscene.instrument_name, "satellite": satscene.fullname } LOG.info("assume seviri-level4") print "... assume seviri-level4" satscene.add_to_history("hdf5 data read by mpop/msg_seviri_hdf.py") if "reader_level" in kwargs.keys(): reader_level = kwargs["reader_level"] else: reader_level = "seviri-level4" if "RSS" in kwargs.keys(): if kwargs["RSS"]: dt_end = 4 else: dt_end = 12 else: from my_msg_module import check_RSS RSS = check_RSS(satscene.sat_nr(), satscene.time_slot) if RSS == None: print "*** Error in mpop/satin/msg_seviri_hdf.py" print " satellite MSG", satscene.sat_nr(), " is not active yet" quit() else: if RSS: dt_end = 4 else: dt_end = 12 print "... hdf file name is specified by observation end time" print " assume ", dt_end, " min between start and end time of observation" # end of scan time 4 min after start end_time = satscene.time_slot + datetime.timedelta(minutes=dt_end) filename = os.path.join( end_time.strftime(conf.get(reader_level, "dir", raw=True)), end_time.strftime(conf.get(reader_level, "filename", raw=True)) % values) print "... search for file: ", filename filenames = glob(str(filename)) if len(filenames) == 0: print "*** Error, no file found" return # just return without exit the program elif len(filenames) > 1: print "*** Warning, more than 1 datafile found: ", filenames filename = filenames[0] print("... read data from %s" % str(filename)) # read data from hdf5 file data_folder = 'U-MARF/MSG/Level1.5/' # Load data from hdf file with h5py.File(filename, 'r') as hf: subset_info = hf.get(data_folder + 'METADATA/SUBSET') for i in range(subset_info.len()): #print subset_info[i]['EntryName'], subset_info[i]['Value'] if subset_info[i][ 'EntryName'] == "VIS_IRSouthLineSelectedRectangle": VIS_IRSouthLine = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "VIS_IRNorthLineSelectedRectangle": VIS_IRNorthLine = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "VIS_IREastColumnSelectedRectangle": VIS_IREastColumn = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "VIS_IRWestColumnSelectedRectangle": VIS_IRWestColumn = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "HRVLowerNorthLineSelectedRectangle": HRVLowerNorthLine = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "HRVLowerSouthLineSelectedRectangle": HRVLowerSouthLine = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "HRVLowerEastColumnSelectedRectangle": HRVLowerEastColumn = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "HRVLowerWestColumnSelectedRectangle": HRVLowerWestColumn = int(subset_info[i]['Value']) if subset_info[i][ 'EntryName'] == "HRVUpperSouthLineSelectedRectangle": HRVUpperSouthLine = int(subset_info[i]['Value']) # 0 if subset_info[i][ 'EntryName'] == "HRVUpperNorthLineSelectedRectangle": HRVUpperNorthLine = int(subset_info[i]['Value']) # 0 if subset_info[i][ 'EntryName'] == "HRVUpperEastColumnSelectedRectangle": HRVUpperEastColumn = int(subset_info[i]['Value']) # 0 if subset_info[i][ 'EntryName'] == "HRVUpperWestColumnSelectedRectangle": HRVUpperWestColumn = int(subset_info[i]['Value']) # 0 sat_status = hf.get( data_folder + 'METADATA/HEADER/SatelliteStatus/SatelliteStatus_DESCR') for i in range(subset_info.len()): if sat_status[i][ 'EntryName'] == "SatelliteDefinition-NominalLongitude": sat_lon = sat_status[i]['Value'] break #print 'VIS_IRSouthLine', VIS_IRSouthLine #print 'VIS_IRNorthLine', VIS_IRNorthLine #print 'VIS_IREastColumn', VIS_IREastColumn #print 'VIS_IRWestColumn', VIS_IRWestColumn #print 'sat_longitude', sat_lon, type(sat_lon), 'GEOS<'+'{:+06.1f}'.format(sat_lon)+'>' if 1 == 0: # works only if all pixels are on the disk from msg_pixcoord2area import msg_pixcoord2area print "VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn: ", VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn area_def = msg_pixcoord2area(VIS_IRNorthLine, VIS_IRWestColumn, VIS_IRSouthLine, VIS_IREastColumn, "vis", sat_lon) else: # works also for pixels outside of the disk pname = 'GEOS<' + '{:+06.1f}'.format( sat_lon) + '>' # "GEOS<+009.5>" proj = { 'proj': 'geos', 'a': '6378169.0', 'b': '6356583.8', 'h': '35785831.0', 'lon_0': str(sat_lon) } aex = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612) # define full disk projection from pyresample.geometry import AreaDefinition full_disk_def = AreaDefinition('full_disk', 'full_disk', pname, proj, 3712, 3712, aex) # define name and calculate area for sub-demain area_name = 'MSG_' + '{:04d}'.format( VIS_IRNorthLine) + '_' + '{:04d}'.format( VIS_IRWestColumn) + '_' + '{:04d}'.format( VIS_IRSouthLine) + '_' + '{:04d}'.format( VIS_IREastColumn) aex = full_disk_def.get_area_extent_for_subset( 3712 - VIS_IRSouthLine, 3712 - VIS_IRWestColumn, 3712 - VIS_IRNorthLine, 3712 - VIS_IREastColumn) area_def = AreaDefinition( area_name, area_name, pname, proj, (VIS_IRWestColumn - VIS_IREastColumn) + 1, (VIS_IRNorthLine - VIS_IRSouthLine) + 1, aex) #print area_def #print "REGION:", area_def.area_id, "{" #print "\tNAME:\t", area_def.name #print "\tPCS_ID:\t", area_def.proj_id #print ("\tPCS_DEF:\tproj="+area_def.proj_dict['proj']+", lon_0=" + area_def.proj_dict['lon_0'] + ", a="+area_def.proj_dict['a']+", b="+area_def.proj_dict['b']+", h="+area_def.proj_dict['h']) #print "\tXSIZE:\t", area_def.x_size #print "\tYSIZE:\t", area_def.y_size #print "\tAREA_EXTENT:\t", area_def.area_extent #print "};" # copy area to satscene satscene.area = area_def # write information used by mipp.xrit.MSG._Calibrator in a fake header file hdr = dict() # satellite ID number hdr["SatelliteDefinition"] = dict() hdr["SatelliteDefinition"]["SatelliteId"] = SatelliteIds[str( satscene.sat_nr())] # processing hdr["Level 1_5 ImageProduction"] = dict() hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"] = np_array( [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], int) # calibration factors Level15ImageCalibration = hf.get( data_folder + 'METADATA/HEADER/RadiometricProcessing/Level15ImageCalibration_ARRAY' ) hdr["Level1_5ImageCalibration"] = dict() for chn_name in channel_numbers.keys(): chn_nb = channel_numbers[chn_name] - 1 hdr["Level1_5ImageCalibration"][chn_nb] = dict() #print chn_name, chn_nb, Level15ImageCalibration[chn_nb]['Cal_Slope'], Level15ImageCalibration[chn_nb]['Cal_Offset'] hdr["Level1_5ImageCalibration"][chn_nb][ 'Cal_Slope'] = Level15ImageCalibration[chn_nb]['Cal_Slope'] hdr["Level1_5ImageCalibration"][chn_nb][ 'Cal_Offset'] = Level15ImageCalibration[chn_nb]['Cal_Offset'] # loop over channels to load for chn_name in satscene.channels_to_load: dataset_name = data_folder + 'DATA/' + dict_channel[ chn_name] + '/IMAGE_DATA' if dataset_name in hf: data_tmp = hf.get(data_folder + 'DATA/' + dict_channel[chn_name] + '/IMAGE_DATA') LOG.info('hdr["SatelliteDefinition"]["SatelliteId"]: ' + str(hdr["SatelliteDefinition"]["SatelliteId"])) #LOG.info('hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"]', hdr["Level 1_5 ImageProduction"]["PlannedChanProcessing"]) chn_nb = channel_numbers[chn_name] - 1 LOG.info( 'hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Slope"]: ' + str(hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Slope"])) LOG.info( 'hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Offset"]: ' + str(hdr["Level1_5ImageCalibration"][chn_nb]["Cal_Offset"])) if calibrate: #Calibrator = _Calibrator(hdr, chn_name) bits_per_pixel = 10 ### !!! I have no idea if this is correct !!! Calibrator = _Calibrator( hdr, chn_name, bits_per_pixel) ## changed call in mipp/xrit/MSG.py data, calibration_unit = Calibrator(data_tmp, calibrate=1) else: data = data_tmp calibration_unit = "counts" LOG.info(chn_name + " min/max: " + str(data.min()) + "," + str(data.max()) + " " + calibration_unit) satscene[chn_name] = ma.asarray(data) satscene[chn_name].info['units'] = calibration_unit satscene[chn_name].info['satname'] = satscene.satname satscene[chn_name].info['satnumber'] = satscene.number satscene[chn_name].info[ 'instrument_name'] = satscene.instrument_name satscene[chn_name].info['time'] = satscene.time_slot satscene[chn_name].info['is_calibrated'] = True else: print "*** Warning, no data for channel " + chn_name + " in file " + filename data = np_nan calibration_unit = "" LOG.info("*** Warning, no data for channel " + chn_name + " in file " + filename)