def get_support_images(input_metadata, sensor, working_dir):
"""
:param input_image:
:param sensor:
:param working_dir:
:return:
"""
time_start = time.time()
display_parameters(locals(), "get_support_images")
supportImages = {}
ratio_l2 = {}
if sensor in ["l8"]:
input_image = searchOneFile(os.path.dirname(input_metadata), "*B1.TIF")
if input_image:
supportImages["R1"] = input_image
else:
print "Missing 30m resolution"
# ratio_l2["R1"] = float(config.L1_resolution) / float(config.L2_resolution)
ratio_l2_coarse = float(config.L1_resolution) / float(config.L2_coarse_resolution)
elif sensor in ["s2"]:
directory_s2 = os.path.join(os.path.dirname(input_metadata), "IMG_DATA")
input_image_r1 = searchOneFile(directory_s2, "*B02.jp2")
if input_image_r1:
supportImages["R1"] = input_image_r1
else:
print "Missing 10m resolution"
input_image_r2 = searchOneFile(directory_s2, "*B05.jp2")
if input_image_r2:
supportImages["R2"] = input_image_r2
else:
print "Missing 20m resolution"
# ratio_l2["R1"] = float(config.S2_L1_resolution) / float(config.S2_R1_resolution)
ratio_l2_coarse = float(config.S2_R1_resolution) / float(config.L2_coarse_resolution)
b1VRT = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_bb.VRT")
b1VRT_up = getBoundingVRT(supportImages["R1"], b1VRT, ratio_l2_coarse)
output = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_coarse.TIF")
#otb_ressampling(supportImages["R1"], output, ratio_l2_coarse)
otb_ressampling(b1VRT_up, output, ratio_l2_coarse)
supportImages["Coarse"] = output
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
logger.info("Generated support images:{}".format(str(supportImages)))
return supportImages
def getS2AgriDEMWBProduct(input_metadata, srtm_dir, working_dir, sensor,
l8_vector, output_directory):
time_start = time.time()
display_parameters_p(locals(), "getS2AgriDEMWBProduct")
if not input_metadata:
print "WARNING ! Missing input metadata !"
return
if sensor == "l8":
path, row = getPathRow(input_metadata)
print "path row", path, row
data_output_dir = os.path.join(output_directory, "Data")
if not os.path.isdir(data_output_dir):
os.makedirs(data_output_dir)
output_product_dir_data = l8_align(os.path.dirname(input_metadata),
l8_vector, data_output_dir,
working_dir, path, row)
keyPathRow = "%03d" % (int(path)) + "%03d" % (int(row))
input_metadata = searchOneFile(output_product_dir_data, "*.txt")
tileId = "L8_TEST_AUX_REFDE2_{}_0001".format(keyPathRow)
elif sensor == 's2':
s2tileId = (os.path.splitext(
os.path.basename(input_metadata))[0]).split("_")[-1][1:]
keyPathRow = s2tileId
tileId = "S2__TEST_AUX_REFDE2_{}____5001".format(s2tileId)
mnt_output_dir = os.path.join(output_directory, "MNT")
if not os.path.isdir(mnt_output_dir):
os.makedirs(mnt_output_dir)
else:
print "existing output mnt", mnt_output_dir
if not os.path.isdir(os.path.join(mnt_output_dir, tileId + ".DBL.DIR")):
outputFiles, supportImages = getDEMWBProduct(input_metadata, srtm_dir,
working_dir, sensor)
mnt_output_dir_dbl_dir = composeMNTProductFromOutputFiles(
outputFiles, mnt_output_dir, tileId)
hdr_filename = os.path.join(mnt_output_dir, tileId + ".HDR")
logging.debug("hdr_filename {}".format(hdr_filename))
if not os.path.isfile(hdr_filename):
hdr_creation(mnt_output_dir_dbl_dir, hdr_filename, sensor)
run_check_mnt(mnt_output_dir, keyPathRow, sensor, supportImages)
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
def checkAllL8S2AgriDEMWBProduct(input_zone):
time_start = time.time()
display_parameters_p(locals(), "getAllL8S2AgriDEMWBProduct")
listdir = os.listdir(input_zone)
rePathRow = "\d{6}$"
inputPathRowDirs = [os.path.join(input_zone,f) for f in listdir if re.search(rePathRow, f) and
os.path.isdir(os.path.join(input_zone,f))]
print inputPathRowDirs
for pathrow in inputPathRowDirs:
data_dir = os.path.join(pathrow, "Data")
if not os.path.isdir(data_dir):
print "Data dir missing"
print "expected {}".format(data_dir)
sys.exit(2)
mnt_up_dir = os.path.join(pathrow, "MNT")
if not os.path.isdir(mnt_up_dir):
print "MNT dir missing"
print "expected {}".format(mnt_up_dir)
sys.exit(2)
mnt_dir = searchOneFile(mnt_up_dir, "*.DBL.DIR")
alt_file = searchOneFile(mnt_dir, "*ALT.TIF")
extent = get_extent(alt_file)
# check all extent
for imageFile in myGlob(mnt_dir, "*.TIF") + myGlob(data_dir, "*/*.TIF"):
extent_ = get_extent(imageFile)
if extent_ != extent:
print "Extent does not match !"
print "Extent of ALT file : {}".format(extent)
print "Extent of {} : {}".format(os.path.basename(imageFile), extent_)
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
def getAllL8S2AgriDEMWBProduct(input_zone, srtm_dir, working_dir, sensor,
l8_vector, output_directory):
time_start = time.time()
display_parameters_p(locals(), "getAllL8S2AgriDEMWBProduct")
listdir = os.listdir(input_zone)
rePathRow = "\d{6}$"
inputPathRowDirs = [
os.path.join(input_zone, f) for f in listdir if
re.search(rePathRow, f) and os.path.isdir(os.path.join(input_zone, f))
]
print inputPathRowDirs
for pathrow in inputPathRowDirs:
wd_pathrow = os.path.join(working_dir, os.path.basename(pathrow))
if not os.path.isdir(wd_pathrow):
os.mkdir(wd_pathrow)
print wd_pathrow
out_pathrow = os.path.join(output_directory, os.path.basename(pathrow))
print out_pathrow
if not os.path.isdir(out_pathrow):
os.mkdir(out_pathrow)
for tile in [os.path.join(pathrow, x) for x in os.listdir(pathrow)]:
metadata = searchOneFile(tile, "*_MTL.txt")
print "metadata", metadata
if not metadata:
print "WARNING ! Missing input metadata"
print "Search path", tile
else:
getS2AgriDEMWBProduct(metadata, srtm_dir, wd_pathrow, "l8",
l8_vector, out_pathrow)
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
def hdr_creation(input_dir, output_filename, sensor):
display_parameters(locals(), "hdr_creation")
idMnt = extractSiteNicknameFromMntDirname(input_dir, sensor)
imageMnt = ""
if sensor == "l8":
count = "7"
files_order = ["SLP", "ASP", "ALT", "ALC", "MSK", "ASC", "SLC"]
mission = "LANDSAT_8"
applicableSite = "L8_TEST_MPL_SITDEF_S_"
imageMNT = searchOneFile(input_dir, "*ALT.TIF")
elif sensor == "s2":
count = "10"
files_order = [
"SLP_R1", "SLP_R2", "ASP_R1", "ASP_R2", "ALT_R1", "ALT_R2", "ALC",
"MSK", "ASC", "SLC"
]
mission = "SENTINEL-2_"
applicableSite = "S2__VM01_MPL_SITDEF_S_" + idMnt
imageMNT = searchOneFile(input_dir, "*ALT_R1.TIF")
info = gdalinfoO(imageMNT)
basename_out = os.path.basename(os.path.splitext(output_filename)[0])
logging.debug(basename_out)
xml_file = os.path.join(os.path.dirname(__file__), "xml_template.xml")
tree = ET.parse(xml_file)
# ET.register_namespace("", "http://eop-cfi.esa.int/CFI")
root = getRoot()
#tree.getroot()
# root = ET.Element("Earth_Explorer_Header", schema_version = "1.00", xmlns = "http://eop-cfi.esa.int/CFI", xsi = "http://www.w3.org/2001/XMLSchema-instance", type = "REFDE2_Header_Type", schemaLocation = "http://eop-cfi.esa.int/CFI AUX_REFDE2_ReferenceDemDataLevel2.xsd")
a1 = ET.SubElement(root, "Fixed_Header")
toto = ET.SubElement(a1, "File_Name")
toto.text = basename_out
toto = ET.SubElement(a1, "File_Description")
toto.text = "ReferenceDemDataLevel2"
toto = ET.SubElement(a1, "Notes")
toto.text = "String"
toto = ET.SubElement(a1, "Mission")
toto.text = mission
toto = ET.SubElement(a1, "File_Class")
toto.text = "TEST"
toto = ET.SubElement(a1, "File_Type")
toto.text = "AUX_REFDE2"
b1 = ET.SubElement(a1, "Validity_Period")
toto = ET.SubElement(b1, "Validity_Start")
toto.text = "UTC=2006-07-01T18:11:45"
toto = ET.SubElement(b1, "Validity_Stop")
toto.text = "UTC=9999-99-99T99:99:99"
toto = ET.SubElement(a1, "File_Version")
toto.text = "0001"
b2 = ET.SubElement(a1, "Source")
toto = ET.SubElement(b2, "System")
toto.text = "MACCS"
toto = ET.SubElement(b2, "Creator")
toto.text = "Earth Explorer CFI"
toto = ET.SubElement(b2, "Creator_Version")
toto.text = "1.1"
toto = ET.SubElement(b2, "Creation_Date")
toto.text = "UTC=2010-02-10T11:00:00"
a = ET.SubElement(root, "Variable_Header")
b2 = ET.SubElement(a, "Main_Product_Header")
ET.SubElement(b2, "List_of_Consumers", count="0")
ET.SubElement(b2, "List_of_Extensions", count="0")
b3 = ET.SubElement(a, "Specific_Product_Header")
b4 = ET.SubElement(b3, "Instance_Id")
toto = ET.SubElement(b4, "Applicable_Site_Nick_Name")
toto.text = idMnt
toto = ET.SubElement(b4, "File_Version")
toto.text = (os.path.basename(input_dir).split("_")[-1]).split(".")[0]
b5 = ET.SubElement(b3, "List_of_Applicable_SiteDefinition_Ids", count="1")
toto = ET.SubElement(b5, "Applicable_SiteDefinition_Id", sn="1")
toto.text = applicableSite
b6 = ET.SubElement(b3, "DEM_Information")
b7 = ET.SubElement(b6, "Cartographic")
b8 = ET.SubElement(b7, "Coordinate_Reference_System")
toto = ET.SubElement(b8, "Code")
toto.text = "EPSG:{}".format(info.getIntEpsgCode())
toto = ET.SubElement(b8, "Short_Description")
toto.text = ""
b8 = ET.SubElement(b7, "Upper_Left_Corner")
toto = ET.SubElement(b8, "X", unit="m")
toto.text = str(info.getOrigin()[0])
toto = ET.SubElement(b8, "Y", unit="m")
toto.text = str(info.getOrigin()[0])
b8 = ET.SubElement(b7, "Sampling_Interval")
toto = ET.SubElement(b8, "By_Line", unit="m")
toto.text = str(info.getPixelSize()[0])
toto = ET.SubElement(b8, "By_Column", unit="m")
toto.text = str(info.getPixelSize()[1])
b8 = ET.SubElement(b7, "Size")
toto = ET.SubElement(b8, "Lines")
toto.text = str(info.getSize()[0])
toto = ET.SubElement(b8, "Columns")
toto.text = str(info.getSize()[0])
toto = ET.SubElement(b6, "Mean_Altitude_Over_L2_Coverage", unit="m")
toto.text = "10"
toto = ET.SubElement(b6,
"Altitude_Standard_Deviation_Over_L2_Coverage",
unit="m")
toto.text = "20"
toto = ET.SubElement(b6, "Nodata_Value")
toto.text = "NIL=N/A"
toto = ET.SubElement(b6, "L2_To_DEM_Subsampling_Ratio")
toto.text = "1"
toto = ET.SubElement(b6, "Comment")
toto.text = ""
# b = ET.SubElement(a, "Specific_Product_Header")
c = ET.SubElement(b3, "DBL_Organization")
d = ET.SubElement(c, "List_of_Packaged_DBL_Files", count=count)
for image in xrange(len(files_order)):
paths_to_file = glob.glob(
os.path.join(input_dir, "*" + files_order[image] + "*"))
if paths_to_file:
path_to_file = paths_to_file[0]
logging.debug(path_to_file)
e1 = ET.SubElement(d, "Packaged_DBL_File", sn=str(image + 1))
ET.SubElement(e1, "Relative_File_Path").text = os.path.join(
os.path.basename(os.path.normpath(input_dir)),
os.path.basename(path_to_file))
ET.SubElement(e1, "File_Definition")
# tree = ET.ElementTree(root)
# output_file = os.sys.argv[1]
logging.debug("############################")
logging.debug(output_filename)
# tree.write(sys.argv[2], encoding = "utf-8", xml_declaration = True, method = "xml")
# xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent = " ")
# with open(output_filename, "w") as f:
# f.write(xmlstr.encode('utf-8'))
tree = ET.ElementTree(root)
f = open(output_filename, "w")
f.write(
ET.tostring(tree,
pretty_print=True,
xml_declaration=True,
encoding="UTF-8"))
f.close()
def l8_align(input_dir, l8_vector, output_directory_path_row_data, working_dir, path=0, row=0):
"""
Runs the crop of all input data
# crop all input tiles to match l8 real extent
:param input_dir:
:param l8_vector:
:param output_data_directory:
:param working_dir:
:param path:
:param row:
:return:
"""
display_parameters(locals(), "l8_align")
time_start = time.time()
extent_wgs84_vector, extent_wgs84, extent_feature = get_extent_from_l8_extent(l8_vector, path, row, working_dir, "4326")
#creates the output data directory containing cropped images (5)
output_product_dir_data = os.path.join(output_directory_path_row_data,
os.path.basename(os.path.normpath(input_dir)))
logging.debug("ouput_product_dir_data {}".format(output_product_dir_data))
if not os.path.isdir(output_product_dir_data):
os.makedirs(output_product_dir_data)
image = searchOneFile(input_dir, "*_B1.TIF")
if not image:
print "B1 is missing"
sys.exit(2)
logger.debug(image)
# txt file
logging.debug(glob.glob(os.path.join(input_dir, "*_MTL.txt")))
txt_file = glob.glob(os.path.join(input_dir, "*_MTL.txt"))[0]
logger.debug(txt_file)
epsg_code = get_image_info_with_gdal(image, get_only_epsg = True)
# print epsg_code
logging.debug("epsg code{}".format(epsg_code))
shape_temp = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_reprojected" + os.path.splitext(os.path.basename(l8_vector))[1])
logger.debug(shape_temp)
#ogr_command = "ogr2ogr -lco \"OGR_ENABLE_PARTIAL_REPROJECTION\"=TRUE -t_srs EPSG:" + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector #
ogr_command = 'ogr2ogr -where \'"PATH"=' + str(path) + ' and "ROW"=' + str(row) + '\' -t_srs EPSG:' + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector
logging.debug(ogr_command)
os.system(ogr_command)
#shape_env = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_env" + os.path.splitext(os.path.basename(l8_vector))[1])
shape_env, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code))
update_txt(txt_file, shape_env_points, output_product_dir_data, epsg_code)
bqa_temp = ""
bqd_out = ""
for tif_file in glob.glob(os.path.join(input_dir, "*.TIF")):
data_out = os.path.join(output_product_dir_data, os.path.basename(tif_file))
resolution = gdalinfoO(tif_file).getPixelSize()[0]
scale_factor = float(config.L2_coarse_resolution)
extent, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code), scale_factor)
if "BQA" in tif_file:
bqa_temp = os.path.join(working_dir, os.path.basename(tif_file))
bqd_out = data_out
crop_with_extent(tif_file, shape_env_points, bqa_temp)
else:
crop_with_extent(tif_file, shape_env_points, data_out)
# crop_with_mask(tif_file, extent, data_out)
if os.path.isfile(bqa_temp):
image_support = searchOneFile(output_product_dir_data, "*_B1.TIF")
if not image_support:
print "B1 is missing in output directory"
sys.exit(2)
bandmath([image_support, bqa_temp], "(im1b1==0&&im2b1==0?0:im2b1)", bqd_out)
else:
logging.info("{} already processed.".format(output_product_dir_data))
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
return output_product_dir_data