def run_matcher():
"""
check matcher
"""
context = { 'gmi': "http://www.isotc211.org/2005/gmi",
'eum': "http://www.eumetsat.int/2008/gmi",
'gco': "http://www.isotc211.org/2005/gco",
'gmd': "http://www.isotc211.org/2005/gmd",
"xsi": "http://www.w3.org/2001/XMLSchema-instance"
}
for the_file in fs.dirwalk('/homespace/gaubert/RODD/src-data/130810-vprodnav/',"*.xml"):
print("file = %s\n" % (the_file))
doc = xml.dom.minidom.parse(the_file).documentElement
lid = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context)
print("[id:%s , path:%s]\n" % (lid[0], the_file))
lid = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context)
print("[id:%s , path:%s]\n" % (lid[0], the_file))
def create_aggregated_viirs_dataset():
dir = "/homespace/gaubert/viirs/Mband-SDR"
file = "SVM01_npp_d20030125_t0847056_e0848301_b00015_c20090513182937523620_gisf_pop.h5"
geo_file = h5py.File("%s/%s" %(dir,"GMODO_npp_d20030125_t0847056_e0848301_b00015_c20090513182937526121_gisf_pop.h5"))
"""
#create a fake file with integer lat lon (on a 16 bits integer lat -90:90 and lon -180:180)
#create type
i16 = numpy.dtype('<i2')
#create empty array
lat_arr = numpy.random.random_integers(-9000,9000, (768,3200))
lat_arr.astype('int16')
lon_arr = numpy.random.random_integers(-18000,18000, (768,3200))
lon_arr.astype('int16')
o_file = h5py.File("/tmp/prototype_file.h5" ,"w")
lat_dset = o_file.create_dataset("Latitude", data=lat_arr, dtype=i16)
lon_dset = o_file.create_dataset("Longitude", data=lon_arr, dtype=i16)
lat_dset = lat_arr[:]
lon_dset = lon_arr[:]
o_file.close()
"""
#create file without geo spatial info
"""
output_file = h5py.File("/tmp/aggreg_sin_geospatial_file.h5" ,"w")
for file in fs_utils.dirwalk(dir,"SVM*.h5"):
input_file = h5py.File(file ,"r")
extract_radiance(output_file, input_file)
output_file.close()
"""
#create file
output_file = h5py.File("/tmp/aggreg_tie_points_file.h5" ,"w")
#create type
f32 = numpy.dtype('<f4')
ui16 = numpy.dtype('<u2')
i16 = numpy.dtype('<i2')
#extract tie-points grid params
(out_lat, out_lon, out_sol_za, out_sol_aa, out_sat_za, out_sat_aa, out_height) = create_tie_points_grid(geo_file)
output_file.create_dataset('Latitude', data = out_lat.astype('float32'), dtype = f32)
output_file.create_dataset('Longitude', data = out_lat.astype('float32'), dtype = f32)
# convert height to i16
numpy.around(out_height, 2, out_height)
out_height = out_height * 100
out_height = out_height.astype('int16')
print("out_height min %d, max %d , range %d\n" % (numpy.min(out_height), numpy.max(out_height), (numpy.max(out_height)- numpy.min(out_height)) ))
output_file.create_dataset('Height', data = out_height.astype('int16'), dtype = i16)
# convert solar zenith angle => Range 0-180
# convert solar azimuth angle => Range -180-180
numpy.around(out_sol_za, 3, out_sol_za)
out_sol_za = out_sol_za * 1000
out_sol_za = out_sol_za.astype('uint16')
print("out_sol_za min %d, max %d , range %d\n" % (numpy.min(out_sol_za), numpy.max(out_sol_za), (numpy.max(out_sol_za)- numpy.min(out_sol_za)) ))
numpy.around(out_sol_aa, 2, out_sol_aa)
out_sol_aa = out_sol_aa * 100
out_sol_aa = out_sol_aa.astype('int16')
print("out_sol_aa min %d, max %d , range %d\n" % (numpy.min(out_sol_aa), numpy.max(out_sol_aa), (numpy.max(out_sol_aa)- numpy.min(out_sol_aa)) ))
# convert sat zenith angle => Range 0-180
# convert sat azimuth angle => Range -180-180
numpy.around(out_sat_za, 3, out_sat_za)
out_sat_za = out_sat_za * 1000
out_sat_za = out_sat_za.astype('uint16')
print("out_sat_za min %d, max %d , range %d\n" % (numpy.min(out_sat_za), numpy.max(out_sat_za), (numpy.max(out_sat_za)- numpy.min(out_sat_za)) ))
numpy.around(out_sat_aa, 2, out_sat_aa)
out_sat_aa = out_sat_aa * 100
out_sat_aa = out_sat_aa.astype('int16')
print("out_sat_aa min %d, max %d , range %d\n" % (numpy.min(out_sat_aa), numpy.max(out_sat_aa), (numpy.max(out_sat_aa)- numpy.min(out_sat_aa)) ))
output_file.create_dataset('SolarZenithAngle', data = out_sol_za, dtype = ui16)
output_file.create_dataset('SolarAzimuthAngle', data = out_sol_aa, dtype = i16)
output_file.create_dataset('SatelliteZenithAngle', data = out_sat_za, dtype = ui16)
output_file.create_dataset('SatelliteAzimuthAngle', data = out_sat_aa, dtype = i16)
extract_extra_geo_spatial_info(output_file, geo_file)
#output_file.create_dataset('SolarZenithAngle', data = out_sol_za, dtype = f32)
#output_file.create_dataset('SolarAzimuthAngle', data = out_sol_aa, dtype = f32)
#output_file.create_dataset('SatelliteZenithAngle', data = out_sat_za, dtype = f32)
#output_file.create_dataset('SatelliteAzimuthAngle', data = out_sat_aa, dtype = f32)
for file in fs_utils.dirwalk(dir,"SVM*.h5"):
input_file = h5py.File(file ,"r")
extract_radiance(output_file, input_file)
output_file.close()
def print_filetype_tree():
"""
Print all filetypes as a tree
"""
context = { 'gmi': "http://www.isotc211.org/2005/gmi",
'eum': "http://www.eumetsat.int/2008/gmi",
'gco': "http://www.isotc211.org/2005/gco",
'gmd': "http://www.isotc211.org/2005/gmd",
"xsi": "http://www.w3.org/2001/XMLSchema-instance"
}
out = StringIO.StringIO()
filtered = StringIO.StringIO()
different_availabilities = set()
for file in fs.dirwalk('/homespace/gaubert/RODD/src-data/130810-vprodnav/',"*.xml"):
#print("file = %s\n" % (file))
doc = xml.dom.minidom.parse(file).documentElement
#doc = elementtree.ElementTree.parse(file)
fileidentifier = xpath.findvalues('/gmi:MI_Metadata/gmd:fileIdentifier/gco:CharacterString', doc, namespaces=context)
#out.write("+-%s:%s\n" % (fileidentifier[0], os.path.basename(file)))
filename_written = False
digitaltransfers = xpath.find('//eum:digitalTransfers/eum:MD_EUMDigitalTransfer', doc, namespaces=context)
for elem in digitaltransfers:
#get availability value
list_of_elems = get_nodes_with("/availability/MD_EUMDigitalTransferOptions/availability/CharacterString", elem.childNodes)
if len(list_of_elems) > 1:
raise Exception("Error too many elements found")
availability_type = " ".join(t.nodeValue for t in list_of_elems[0].childNodes if t.nodeType == t.TEXT_NODE)
different_availabilities.add(availability_type.strip())
# get list of channels
list_of_channels = get_nodes_with("/availability/MD_EUMDigitalTransferOptions/eumetcastChannels/CharacterString", elem.childNodes)
chans = ""
for ch in list_of_channels:
chans += " ".join(t.nodeValue for t in ch.childNodes if t.nodeType == t.TEXT_NODE)
#print("chans = %s\n" %(chans))
if contains(availability_type, ['EUMETCAST','GTS','DIRECT']):
#write name
if not filename_written:
out.write("+-%s:%s:ch=[%s]\n" % (fileidentifier[0], os.path.basename(file),chans))
filename_written = False
# get associated formats to this type
#if contains(availability_type,["EUMETCAST","GEONETCAST", ] ):
format_list = get_nodes_with("/format/MD_EUMFormat", elem.childNodes)
for e in format_list:
dummy_list = get_nodes_with("/name/CharacterString", e.childNodes)
dum_node = dummy_list[0]
name = " ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE)
dummy_list = get_nodes_with("/typicalFilename/CharacterString", e.childNodes)
typicalfilenames = []
for dum_node in dummy_list:
typicalfilenames.append(" ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE))
out.write(" \__(%s:%s)\n" % (availability_type, name.strip()) )
for n in typicalfilenames:
out.write(" \__%s\n" % (n.strip()))
else:
#write name
if not filename_written:
filtered.write("+-%s:%s\n" % (fileidentifier[0], os.path.basename(file)))
filename_written = False
# get associated formats to this type
#if contains(availability_type,["EUMETCAST","GEONETCAST", ] ):
format_list = get_nodes_with("/format/MD_EUMFormat", elem.childNodes)
for e in format_list:
dummy_list = get_nodes_with("/name/CharacterString", e.childNodes)
dum_node = dummy_list[0]
name = " ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE)
dummy_list = get_nodes_with("/typicalFilename/CharacterString", e.childNodes)
typicalfilenames = []
for dum_node in dummy_list:
typicalfilenames.append(" ".join(t.nodeValue for t in dum_node.childNodes if t.nodeType == t.TEXT_NODE))
filtered.write(" \__(%s:%s)\n" % (availability_type, name.strip()) )
for n in typicalfilenames:
filtered.write(" \__%s\n" % (n.strip()))
#print("name = %s ; filesnames = %s\n" % (name, typicalfilenames) )
#print(out.getvalue())
out.write("-------------------------------------------------------------------\n")
out.write("Availabilities type:\n")
for av in different_availabilities:
out.write("- %s\n" % av)
o_file= open("/tmp/dissemination-tree.txt", "w")
o_file.write(out.getvalue())
o_file.close()
o_file= open("/tmp/filtered-tree.txt", "w")
o_file.write(filtered.getvalue())
o_file.close()
