def run_matcher(): """ check matcher """ context = { 'gmi': "http://www.isotc211.org/2005/gmi", 'eum': "http://www.eumetsat.int/2008/gmi", 'gco': "http://www.isotc211.org/2005/gco", 'gmd': "http://www.isotc211.org/2005/gmd", "xsi": "http://www.w3.org/2001/XMLSchema-instance" } for the_file in fs.dirwalk('/homespace/gaubert/RODD/src-data/130810-vprodnav/',"*.xml"): print("file = %s\n" % (the_file)) doc = xml.dom.minidom.parse(the_file).documentElement lid = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context) print("[id:%s , path:%s]\n" % (lid[0], the_file)) lid = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context) print("[id:%s , path:%s]\n" % (lid[0], the_file))
def create_aggregated_viirs_dataset(): dir = "/homespace/gaubert/viirs/Mband-SDR" file = "SVM01_npp_d20030125_t0847056_e0848301_b00015_c20090513182937523620_gisf_pop.h5" geo_file = h5py.File("%s/%s" %(dir,"GMODO_npp_d20030125_t0847056_e0848301_b00015_c20090513182937526121_gisf_pop.h5")) """ #create a fake file with integer lat lon (on a 16 bits integer lat -90:90 and lon -180:180) #create type i16 = numpy.dtype('<i2') #create empty array lat_arr = numpy.random.random_integers(-9000,9000, (768,3200)) lat_arr.astype('int16') lon_arr = numpy.random.random_integers(-18000,18000, (768,3200)) lon_arr.astype('int16') o_file = h5py.File("/tmp/prototype_file.h5" ,"w") lat_dset = o_file.create_dataset("Latitude", data=lat_arr, dtype=i16) lon_dset = o_file.create_dataset("Longitude", data=lon_arr, dtype=i16) lat_dset = lat_arr[:] lon_dset = lon_arr[:] o_file.close() """ #create file without geo spatial info """ output_file = h5py.File("/tmp/aggreg_sin_geospatial_file.h5" ,"w") for file in fs_utils.dirwalk(dir,"SVM*.h5"): input_file = h5py.File(file ,"r") extract_radiance(output_file, input_file) output_file.close() """ #create file output_file = h5py.File("/tmp/aggreg_tie_points_file.h5" ,"w") #create type f32 = numpy.dtype('<f4') ui16 = numpy.dtype('<u2') i16 = numpy.dtype('<i2') #extract tie-points grid params (out_lat, out_lon, out_sol_za, out_sol_aa, out_sat_za, out_sat_aa, out_height) = create_tie_points_grid(geo_file) output_file.create_dataset('Latitude', data = out_lat.astype('float32'), dtype = f32) output_file.create_dataset('Longitude', data = out_lat.astype('float32'), dtype = f32) # convert height to i16 numpy.around(out_height, 2, out_height) out_height = out_height * 100 out_height = out_height.astype('int16') print("out_height min %d, max %d , range %d\n" % (numpy.min(out_height), numpy.max(out_height), (numpy.max(out_height)- numpy.min(out_height)) )) output_file.create_dataset('Height', data = out_height.astype('int16'), dtype = i16) # convert solar zenith angle => Range 0-180 # convert solar azimuth angle => Range -180-180 numpy.around(out_sol_za, 3, out_sol_za) out_sol_za = out_sol_za * 1000 out_sol_za = out_sol_za.astype('uint16') print("out_sol_za min %d, max %d , range %d\n" % (numpy.min(out_sol_za), numpy.max(out_sol_za), (numpy.max(out_sol_za)- numpy.min(out_sol_za)) )) numpy.around(out_sol_aa, 2, out_sol_aa) out_sol_aa = out_sol_aa * 100 out_sol_aa = out_sol_aa.astype('int16') print("out_sol_aa min %d, max %d , range %d\n" % (numpy.min(out_sol_aa), numpy.max(out_sol_aa), (numpy.max(out_sol_aa)- numpy.min(out_sol_aa)) )) # convert sat zenith angle => Range 0-180 # convert sat azimuth angle => Range -180-180 numpy.around(out_sat_za, 3, out_sat_za) out_sat_za = out_sat_za * 1000 out_sat_za = out_sat_za.astype('uint16') print("out_sat_za min %d, max %d , range %d\n" % (numpy.min(out_sat_za), numpy.max(out_sat_za), (numpy.max(out_sat_za)- numpy.min(out_sat_za)) )) numpy.around(out_sat_aa, 2, out_sat_aa) out_sat_aa = out_sat_aa * 100 out_sat_aa = out_sat_aa.astype('int16') print("out_sat_aa min %d, max %d , range %d\n" % (numpy.min(out_sat_aa), numpy.max(out_sat_aa), (numpy.max(out_sat_aa)- numpy.min(out_sat_aa)) )) output_file.create_dataset('SolarZenithAngle', data = out_sol_za, dtype = ui16) output_file.create_dataset('SolarAzimuthAngle', data = out_sol_aa, dtype = i16) output_file.create_dataset('SatelliteZenithAngle', data = out_sat_za, dtype = ui16) output_file.create_dataset('SatelliteAzimuthAngle', data = out_sat_aa, dtype = i16) extract_extra_geo_spatial_info(output_file, geo_file) #output_file.create_dataset('SolarZenithAngle', data = out_sol_za, dtype = f32) #output_file.create_dataset('SolarAzimuthAngle', data = out_sol_aa, dtype = f32) #output_file.create_dataset('SatelliteZenithAngle', data = out_sat_za, dtype = f32) #output_file.create_dataset('SatelliteAzimuthAngle', data = out_sat_aa, dtype = f32) for file in fs_utils.dirwalk(dir,"SVM*.h5"): input_file = h5py.File(file ,"r") extract_radiance(output_file, input_file) output_file.close()
def print_filetype_tree(): """ Print all filetypes as a tree """ context = { 'gmi': "http://www.isotc211.org/2005/gmi", 'eum': "http://www.eumetsat.int/2008/gmi", 'gco': "http://www.isotc211.org/2005/gco", 'gmd': "http://www.isotc211.org/2005/gmd", "xsi": "http://www.w3.org/2001/XMLSchema-instance" } out = StringIO.StringIO() filtered = StringIO.StringIO() different_availabilities = set() for file in fs.dirwalk('/homespace/gaubert/RODD/src-data/130810-vprodnav/',"*.xml"): #print("file = %s\n" % (file)) doc = xml.dom.minidom.parse(file).documentElement #doc = elementtree.ElementTree.parse(file) fileidentifier = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context) #out.write("+-%s:%s\n" % (fileidentifier[0], os.path.basename(file))) filename_written = False digitaltransfers = xpath.find('//eum:digitalTransfers/eum:MD_EUMDigitalTransfer', doc, namespaces=context) for elem in digitaltransfers: #get availability value list_of_elems = get_nodes_with("/availability/MD_EUMDigitalTransferOptions/availability/CharacterString", elem.childNodes) if len(list_of_elems) > 1: raise Exception("Error too many elements found") availability_type = " ".join(t.nodeValue for t in list_of_elems[0].childNodes if t.nodeType == t.TEXT_NODE) different_availabilities.add(availability_type.strip()) # get list of channels list_of_channels = get_nodes_with("/availability/MD_EUMDigitalTransferOptions/eumetcastChannels/CharacterString", elem.childNodes) chans = "" for ch in list_of_channels: chans += " ".join(t.nodeValue for t in ch.childNodes if t.nodeType == t.TEXT_NODE) #print("chans = %s\n" %(chans)) if contains(availability_type, ['EUMETCAST','GTS','DIRECT']): #write name if not filename_written: out.write("+-%s:%s:ch=[%s]\n" % (fileidentifier[0], os.path.basename(file),chans)) filename_written = False # get associated formats to this type #if contains(availability_type,["EUMETCAST","GEONETCAST", ] ): format_list = get_nodes_with("/format/MD_EUMFormat", elem.childNodes) for e in format_list: dummy_list = get_nodes_with("/name/CharacterString", e.childNodes) dum_node = dummy_list[0] name = " ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE) dummy_list = get_nodes_with("/typicalFilename/CharacterString", e.childNodes) typicalfilenames = [] for dum_node in dummy_list: typicalfilenames.append(" ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE)) out.write(" \__(%s:%s)\n" % (availability_type, name.strip()) ) for n in typicalfilenames: out.write(" \__%s\n" % (n.strip())) else: #write name if not filename_written: filtered.write("+-%s:%s\n" % (fileidentifier[0], os.path.basename(file))) filename_written = False # get associated formats to this type #if contains(availability_type,["EUMETCAST","GEONETCAST", ] ): format_list = get_nodes_with("/format/MD_EUMFormat", elem.childNodes) for e in format_list: dummy_list = get_nodes_with("/name/CharacterString", e.childNodes) dum_node = dummy_list[0] name = " ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE) dummy_list = get_nodes_with("/typicalFilename/CharacterString", e.childNodes) typicalfilenames = [] for dum_node in dummy_list: typicalfilenames.append(" ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE)) filtered.write(" \__(%s:%s)\n" % (availability_type, name.strip()) ) for n in typicalfilenames: filtered.write(" \__%s\n" % (n.strip())) #print("name = %s ; filesnames = %s\n" % (name, typicalfilenames) ) #print(out.getvalue()) out.write("-------------------------------------------------------------------\n") out.write("Availabilities type:\n") for av in different_availabilities: out.write("- %s\n" % av) o_file= open("/tmp/dissemination-tree.txt", "w") o_file.write(out.getvalue()) o_file.close() o_file= open("/tmp/filtered-tree.txt", "w") o_file.write(filtered.getvalue()) o_file.close()