def get_support_images(input_metadata, sensor, working_dir): """ :param input_image: :param sensor: :param working_dir: :return: """ time_start = time.time() display_parameters(locals(), "get_support_images") supportImages = {} ratio_l2 = {} if sensor in ["l8"]: input_image = searchOneFile(os.path.dirname(input_metadata), "*B1.TIF") if input_image: supportImages["R1"] = input_image else: print "Missing 30m resolution" # ratio_l2["R1"] = float(config.L1_resolution) / float(config.L2_resolution) ratio_l2_coarse = float(config.L1_resolution) / float(config.L2_coarse_resolution) elif sensor in ["s2"]: directory_s2 = os.path.join(os.path.dirname(input_metadata), "IMG_DATA") input_image_r1 = searchOneFile(directory_s2, "*B02.jp2") if input_image_r1: supportImages["R1"] = input_image_r1 else: print "Missing 10m resolution" input_image_r2 = searchOneFile(directory_s2, "*B05.jp2") if input_image_r2: supportImages["R2"] = input_image_r2 else: print "Missing 20m resolution" # ratio_l2["R1"] = float(config.S2_L1_resolution) / float(config.S2_R1_resolution) ratio_l2_coarse = float(config.S2_R1_resolution) / float(config.L2_coarse_resolution) b1VRT = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_bb.VRT") b1VRT_up = getBoundingVRT(supportImages["R1"], b1VRT, ratio_l2_coarse) output = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_coarse.TIF") #otb_ressampling(supportImages["R1"], output, ratio_l2_coarse) otb_ressampling(b1VRT_up, output, ratio_l2_coarse) supportImages["Coarse"] = output interval = time.time() - time_start logger.info('Total time in seconds:' + str(interval)) logger.info("Generated support images:{}".format(str(supportImages))) return supportImages
def getS2AgriDEMWBProduct(input_metadata, srtm_dir, working_dir, sensor, l8_vector, output_directory): time_start = time.time() display_parameters_p(locals(), "getS2AgriDEMWBProduct") if not input_metadata: print "WARNING ! Missing input metadata !" return if sensor == "l8": path, row = getPathRow(input_metadata) print "path row", path, row data_output_dir = os.path.join(output_directory, "Data") if not os.path.isdir(data_output_dir): os.makedirs(data_output_dir) output_product_dir_data = l8_align(os.path.dirname(input_metadata), l8_vector, data_output_dir, working_dir, path, row) keyPathRow = "%03d" % (int(path)) + "%03d" % (int(row)) input_metadata = searchOneFile(output_product_dir_data, "*.txt") tileId = "L8_TEST_AUX_REFDE2_{}_0001".format(keyPathRow) elif sensor == 's2': s2tileId = (os.path.splitext( os.path.basename(input_metadata))[0]).split("_")[-1][1:] keyPathRow = s2tileId tileId = "S2__TEST_AUX_REFDE2_{}____5001".format(s2tileId) mnt_output_dir = os.path.join(output_directory, "MNT") if not os.path.isdir(mnt_output_dir): os.makedirs(mnt_output_dir) else: print "existing output mnt", mnt_output_dir if not os.path.isdir(os.path.join(mnt_output_dir, tileId + ".DBL.DIR")): outputFiles, supportImages = getDEMWBProduct(input_metadata, srtm_dir, working_dir, sensor) mnt_output_dir_dbl_dir = composeMNTProductFromOutputFiles( outputFiles, mnt_output_dir, tileId) hdr_filename = os.path.join(mnt_output_dir, tileId + ".HDR") logging.debug("hdr_filename {}".format(hdr_filename)) if not os.path.isfile(hdr_filename): hdr_creation(mnt_output_dir_dbl_dir, hdr_filename, sensor) run_check_mnt(mnt_output_dir, keyPathRow, sensor, supportImages) interval = time.time() - time_start logger.info('Total time in seconds:' + str(interval))
def checkAllL8S2AgriDEMWBProduct(input_zone): time_start = time.time() display_parameters_p(locals(), "getAllL8S2AgriDEMWBProduct") listdir = os.listdir(input_zone) rePathRow = "\d{6}$" inputPathRowDirs = [os.path.join(input_zone,f) for f in listdir if re.search(rePathRow, f) and os.path.isdir(os.path.join(input_zone,f))] print inputPathRowDirs for pathrow in inputPathRowDirs: data_dir = os.path.join(pathrow, "Data") if not os.path.isdir(data_dir): print "Data dir missing" print "expected {}".format(data_dir) sys.exit(2) mnt_up_dir = os.path.join(pathrow, "MNT") if not os.path.isdir(mnt_up_dir): print "MNT dir missing" print "expected {}".format(mnt_up_dir) sys.exit(2) mnt_dir = searchOneFile(mnt_up_dir, "*.DBL.DIR") alt_file = searchOneFile(mnt_dir, "*ALT.TIF") extent = get_extent(alt_file) # check all extent for imageFile in myGlob(mnt_dir, "*.TIF") + myGlob(data_dir, "*/*.TIF"): extent_ = get_extent(imageFile) if extent_ != extent: print "Extent does not match !" print "Extent of ALT file : {}".format(extent) print "Extent of {} : {}".format(os.path.basename(imageFile), extent_) interval = time.time() - time_start logger.info('Total time in seconds:' + str(interval))
def getAllL8S2AgriDEMWBProduct(input_zone, srtm_dir, working_dir, sensor, l8_vector, output_directory): time_start = time.time() display_parameters_p(locals(), "getAllL8S2AgriDEMWBProduct") listdir = os.listdir(input_zone) rePathRow = "\d{6}$" inputPathRowDirs = [ os.path.join(input_zone, f) for f in listdir if re.search(rePathRow, f) and os.path.isdir(os.path.join(input_zone, f)) ] print inputPathRowDirs for pathrow in inputPathRowDirs: wd_pathrow = os.path.join(working_dir, os.path.basename(pathrow)) if not os.path.isdir(wd_pathrow): os.mkdir(wd_pathrow) print wd_pathrow out_pathrow = os.path.join(output_directory, os.path.basename(pathrow)) print out_pathrow if not os.path.isdir(out_pathrow): os.mkdir(out_pathrow) for tile in [os.path.join(pathrow, x) for x in os.listdir(pathrow)]: metadata = searchOneFile(tile, "*_MTL.txt") print "metadata", metadata if not metadata: print "WARNING ! Missing input metadata" print "Search path", tile else: getS2AgriDEMWBProduct(metadata, srtm_dir, wd_pathrow, "l8", l8_vector, out_pathrow) interval = time.time() - time_start logger.info('Total time in seconds:' + str(interval))
def hdr_creation(input_dir, output_filename, sensor): display_parameters(locals(), "hdr_creation") idMnt = extractSiteNicknameFromMntDirname(input_dir, sensor) imageMnt = "" if sensor == "l8": count = "7" files_order = ["SLP", "ASP", "ALT", "ALC", "MSK", "ASC", "SLC"] mission = "LANDSAT_8" applicableSite = "L8_TEST_MPL_SITDEF_S_" imageMNT = searchOneFile(input_dir, "*ALT.TIF") elif sensor == "s2": count = "10" files_order = [ "SLP_R1", "SLP_R2", "ASP_R1", "ASP_R2", "ALT_R1", "ALT_R2", "ALC", "MSK", "ASC", "SLC" ] mission = "SENTINEL-2_" applicableSite = "S2__VM01_MPL_SITDEF_S_" + idMnt imageMNT = searchOneFile(input_dir, "*ALT_R1.TIF") info = gdalinfoO(imageMNT) basename_out = os.path.basename(os.path.splitext(output_filename)[0]) logging.debug(basename_out) xml_file = os.path.join(os.path.dirname(__file__), "xml_template.xml") tree = ET.parse(xml_file) # ET.register_namespace("", "http://eop-cfi.esa.int/CFI") root = getRoot() #tree.getroot() # root = ET.Element("Earth_Explorer_Header", schema_version = "1.00", xmlns = "http://eop-cfi.esa.int/CFI", xsi = "http://www.w3.org/2001/XMLSchema-instance", type = "REFDE2_Header_Type", schemaLocation = "http://eop-cfi.esa.int/CFI AUX_REFDE2_ReferenceDemDataLevel2.xsd") a1 = ET.SubElement(root, "Fixed_Header") toto = ET.SubElement(a1, "File_Name") toto.text = basename_out toto = ET.SubElement(a1, "File_Description") toto.text = "ReferenceDemDataLevel2" toto = ET.SubElement(a1, "Notes") toto.text = "String" toto = ET.SubElement(a1, "Mission") toto.text = mission toto = ET.SubElement(a1, "File_Class") toto.text = "TEST" toto = ET.SubElement(a1, "File_Type") toto.text = "AUX_REFDE2" b1 = ET.SubElement(a1, "Validity_Period") toto = ET.SubElement(b1, "Validity_Start") toto.text = "UTC=2006-07-01T18:11:45" toto = ET.SubElement(b1, "Validity_Stop") toto.text = "UTC=9999-99-99T99:99:99" toto = ET.SubElement(a1, "File_Version") toto.text = "0001" b2 = ET.SubElement(a1, "Source") toto = ET.SubElement(b2, "System") toto.text = "MACCS" toto = ET.SubElement(b2, "Creator") toto.text = "Earth Explorer CFI" toto = ET.SubElement(b2, "Creator_Version") toto.text = "1.1" toto = ET.SubElement(b2, "Creation_Date") toto.text = "UTC=2010-02-10T11:00:00" a = ET.SubElement(root, "Variable_Header") b2 = ET.SubElement(a, "Main_Product_Header") ET.SubElement(b2, "List_of_Consumers", count="0") ET.SubElement(b2, "List_of_Extensions", count="0") b3 = ET.SubElement(a, "Specific_Product_Header") b4 = ET.SubElement(b3, "Instance_Id") toto = ET.SubElement(b4, "Applicable_Site_Nick_Name") toto.text = idMnt toto = ET.SubElement(b4, "File_Version") toto.text = (os.path.basename(input_dir).split("_")[-1]).split(".")[0] b5 = ET.SubElement(b3, "List_of_Applicable_SiteDefinition_Ids", count="1") toto = ET.SubElement(b5, "Applicable_SiteDefinition_Id", sn="1") toto.text = applicableSite b6 = ET.SubElement(b3, "DEM_Information") b7 = ET.SubElement(b6, "Cartographic") b8 = ET.SubElement(b7, "Coordinate_Reference_System") toto = ET.SubElement(b8, "Code") toto.text = "EPSG:{}".format(info.getIntEpsgCode()) toto = ET.SubElement(b8, "Short_Description") toto.text = "" b8 = ET.SubElement(b7, "Upper_Left_Corner") toto = ET.SubElement(b8, "X", unit="m") toto.text = str(info.getOrigin()[0]) toto = ET.SubElement(b8, "Y", unit="m") toto.text = str(info.getOrigin()[0]) b8 = ET.SubElement(b7, "Sampling_Interval") toto = ET.SubElement(b8, "By_Line", unit="m") toto.text = str(info.getPixelSize()[0]) toto = ET.SubElement(b8, "By_Column", unit="m") toto.text = str(info.getPixelSize()[1]) b8 = ET.SubElement(b7, "Size") toto = ET.SubElement(b8, "Lines") toto.text = str(info.getSize()[0]) toto = ET.SubElement(b8, "Columns") toto.text = str(info.getSize()[0]) toto = ET.SubElement(b6, "Mean_Altitude_Over_L2_Coverage", unit="m") toto.text = "10" toto = ET.SubElement(b6, "Altitude_Standard_Deviation_Over_L2_Coverage", unit="m") toto.text = "20" toto = ET.SubElement(b6, "Nodata_Value") toto.text = "NIL=N/A" toto = ET.SubElement(b6, "L2_To_DEM_Subsampling_Ratio") toto.text = "1" toto = ET.SubElement(b6, "Comment") toto.text = "" # b = ET.SubElement(a, "Specific_Product_Header") c = ET.SubElement(b3, "DBL_Organization") d = ET.SubElement(c, "List_of_Packaged_DBL_Files", count=count) for image in xrange(len(files_order)): paths_to_file = glob.glob( os.path.join(input_dir, "*" + files_order[image] + "*")) if paths_to_file: path_to_file = paths_to_file[0] logging.debug(path_to_file) e1 = ET.SubElement(d, "Packaged_DBL_File", sn=str(image + 1)) ET.SubElement(e1, "Relative_File_Path").text = os.path.join( os.path.basename(os.path.normpath(input_dir)), os.path.basename(path_to_file)) ET.SubElement(e1, "File_Definition") # tree = ET.ElementTree(root) # output_file = os.sys.argv[1] logging.debug("############################") logging.debug(output_filename) # tree.write(sys.argv[2], encoding = "utf-8", xml_declaration = True, method = "xml") # xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent = " ") # with open(output_filename, "w") as f: # f.write(xmlstr.encode('utf-8')) tree = ET.ElementTree(root) f = open(output_filename, "w") f.write( ET.tostring(tree, pretty_print=True, xml_declaration=True, encoding="UTF-8")) f.close()
def l8_align(input_dir, l8_vector, output_directory_path_row_data, working_dir, path=0, row=0): """ Runs the crop of all input data # crop all input tiles to match l8 real extent :param input_dir: :param l8_vector: :param output_data_directory: :param working_dir: :param path: :param row: :return: """ display_parameters(locals(), "l8_align") time_start = time.time() extent_wgs84_vector, extent_wgs84, extent_feature = get_extent_from_l8_extent(l8_vector, path, row, working_dir, "4326") #creates the output data directory containing cropped images (5) output_product_dir_data = os.path.join(output_directory_path_row_data, os.path.basename(os.path.normpath(input_dir))) logging.debug("ouput_product_dir_data {}".format(output_product_dir_data)) if not os.path.isdir(output_product_dir_data): os.makedirs(output_product_dir_data) image = searchOneFile(input_dir, "*_B1.TIF") if not image: print "B1 is missing" sys.exit(2) logger.debug(image) # txt file logging.debug(glob.glob(os.path.join(input_dir, "*_MTL.txt"))) txt_file = glob.glob(os.path.join(input_dir, "*_MTL.txt"))[0] logger.debug(txt_file) epsg_code = get_image_info_with_gdal(image, get_only_epsg = True) # print epsg_code logging.debug("epsg code{}".format(epsg_code)) shape_temp = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_reprojected" + os.path.splitext(os.path.basename(l8_vector))[1]) logger.debug(shape_temp) #ogr_command = "ogr2ogr -lco \"OGR_ENABLE_PARTIAL_REPROJECTION\"=TRUE -t_srs EPSG:" + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector # ogr_command = 'ogr2ogr -where \'"PATH"=' + str(path) + ' and "ROW"=' + str(row) + '\' -t_srs EPSG:' + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector logging.debug(ogr_command) os.system(ogr_command) #shape_env = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_env" + os.path.splitext(os.path.basename(l8_vector))[1]) shape_env, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code)) update_txt(txt_file, shape_env_points, output_product_dir_data, epsg_code) bqa_temp = "" bqd_out = "" for tif_file in glob.glob(os.path.join(input_dir, "*.TIF")): data_out = os.path.join(output_product_dir_data, os.path.basename(tif_file)) resolution = gdalinfoO(tif_file).getPixelSize()[0] scale_factor = float(config.L2_coarse_resolution) extent, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code), scale_factor) if "BQA" in tif_file: bqa_temp = os.path.join(working_dir, os.path.basename(tif_file)) bqd_out = data_out crop_with_extent(tif_file, shape_env_points, bqa_temp) else: crop_with_extent(tif_file, shape_env_points, data_out) # crop_with_mask(tif_file, extent, data_out) if os.path.isfile(bqa_temp): image_support = searchOneFile(output_product_dir_data, "*_B1.TIF") if not image_support: print "B1 is missing in output directory" sys.exit(2) bandmath([image_support, bqa_temp], "(im1b1==0&&im2b1==0?0:im2b1)", bqd_out) else: logging.info("{} already processed.".format(output_product_dir_data)) interval = time.time() - time_start logger.info('Total time in seconds:' + str(interval)) return output_product_dir_data