def get_arguments():
"""
Manages inputs
:return:
"""
# ####### MANAGING ARGUMENTS ########
# TODO : set path to downloadwb as an argument
# check inputs
logger.info("######## MANAGING ARGUMENTS ########")
logger.debug("Checking arguments")
argParser = argparse.ArgumentParser()
argParser.add_argument('-i',
'--input_directory',
help='Path to input directory')
argParser.add_argument('-z',
'--input_zone_directory',
help='Path to input directory')
argParser.add_argument('-s',
"--sensor",
choices=['s2', 'l8'],
required=True)
input_directory = input_zone = None
logger.debug(len(sys.argv))
if len(sys.argv) != 3:
print "Nb args = ", len(sys.argv), " required: 3."
usage(argParser)
args = argParser.parse_args(sys.argv[1:])
input_directory = args.input_directory
input_zone = args.input_zone_directory
sensor = args.sensor
logger.debug("Arguments ok")
if input_directory:
input_directory = os.path.realpath(input_directory)
# creating directories if they don't exist
logger.debug("Managing input directories")
if not os.path.isdir(input_directory):
usage(argParser)
sys.exit(2)
if input_zone:
input_zone = os.path.realpath(input_zone)
# creating directories if they don't exist
logger.debug("Managing input directories")
if not os.path.isdir(input_zone):
usage(argParser)
sys.exit(2)
display_parameters(locals(), "checkdem_get_arguments")
return input_directory, input_zone, sensor
def run_check_mnt(MNTDir, pathRow, sensor, supportImages=None):
display_parameters(locals(), "run_check_mnt")
if not os.path.isdir(MNTDir):
print "Bad structure"
print "Required:", MNTDir
#checking mnt
mntFile, mntDir = checkMNTStructure(MNTDir, os.path.basename(pathRow),
sensor)
extent = get_extent(mntFile)
print "\t Checking on extent", extent
checkExtent(mntDir, extent, supportImages)
def get_support_images(input_metadata, sensor, working_dir):
"""
:param input_image:
:param sensor:
:param working_dir:
:return:
"""
time_start = time.time()
display_parameters(locals(), "get_support_images")
supportImages = {}
ratio_l2 = {}
if sensor in ["l8"]:
input_image = searchOneFile(os.path.dirname(input_metadata), "*B1.TIF")
if input_image:
supportImages["R1"] = input_image
else:
print "Missing 30m resolution"
# ratio_l2["R1"] = float(config.L1_resolution) / float(config.L2_resolution)
ratio_l2_coarse = float(config.L1_resolution) / float(config.L2_coarse_resolution)
elif sensor in ["s2"]:
directory_s2 = os.path.join(os.path.dirname(input_metadata), "IMG_DATA")
input_image_r1 = searchOneFile(directory_s2, "*B02.jp2")
if input_image_r1:
supportImages["R1"] = input_image_r1
else:
print "Missing 10m resolution"
input_image_r2 = searchOneFile(directory_s2, "*B05.jp2")
if input_image_r2:
supportImages["R2"] = input_image_r2
else:
print "Missing 20m resolution"
# ratio_l2["R1"] = float(config.S2_L1_resolution) / float(config.S2_R1_resolution)
ratio_l2_coarse = float(config.S2_R1_resolution) / float(config.L2_coarse_resolution)
b1VRT = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_bb.VRT")
b1VRT_up = getBoundingVRT(supportImages["R1"], b1VRT, ratio_l2_coarse)
output = os.path.join(working_dir, getBasename(supportImages["R1"]) + "_coarse.TIF")
#otb_ressampling(supportImages["R1"], output, ratio_l2_coarse)
otb_ressampling(b1VRT_up, output, ratio_l2_coarse)
supportImages["Coarse"] = output
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
logger.info("Generated support images:{}".format(str(supportImages)))
return supportImages
def generate_dem_l2(input_mnt, working_dir):
"""
:param input_mnt:
:param working_dir:
:return:
"""
time_start = time.time()
display_parameters(locals(), "get_tile_dem")
slope = get_slope(input_mnt, working_dir)
aspect = get_aspect(input_mnt, working_dir)
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
logger.info("Generated slope: {}".format(slope))
logger.info("Generated aspect:{}".format(aspect))
return slope, aspect
def get_slope(dtm_l2, working_dir, scale=True):
"""
Get slope at dtm_l2 resolution
:param dtm_l2:
:param working_dir:
:param scale:
:return:
"""
display_parameters(locals(), "get_slope")
# running gdaldem to get slope
logger.debug("gdal dem")
slope_raw = os.path.join(working_dir,
getBasename(dtm_l2) + "_slope_raw.tif")
logger.debug("slope file : " + slope_raw)
commandGDAL = "gdaldem slope " + dtm_l2 + " "
if scale:
commandGDAL += " -compute_edges "
commandGDAL += slope_raw
logger.debug("command : " + commandGDAL)
os.system(commandGDAL)
logger.debug("gdal dem ok")
slope_radian = os.path.join(working_dir,
getBasename(dtm_l2) + "_slope_rad.tif")
# la pente est exprimée en degre float 32, on remet en radians * 100 codé sur 16 bits
# les degres vont de 0 à 90, onr emet entre 0 et pi/2*100
cmd = "gdal_translate -ot Int16 -scale 0 90 0 157 " + slope_raw + " " + slope_radian
os.system(cmd)
# Necessary step ?
seuil = -500
output_slope = os.path.join(working_dir,
getBasename(dtm_l2) + "_slope.tif")
bandmath([slope_radian], "(im1b1<" + str(seuil) + "?0:im1b1)",
output_slope)
logger.info("Output Slope")
logger.info("\t Slope : " + output_slope)
return output_slope
def generate_dem_l1(input_mnt, support_image, dtm_output):
"""
:param input_mnt:
:param support_image:
:param dtm_output:
:return:
"""
time_start = time.time()
display_parameters(locals(), "generate_dem_l2")
# superimpose the vrt on the input data
if not os.path.isfile(dtm_output):
superimpose(input_mnt, support_image, dtm_output, 40)
logger.info("Output DTM")
logger.info("\t DTM L1 resolution : " + dtm_output)
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
logger.info("Generated raw mnt: {}".format(dtm_output))
def get_aspect(dtm_l2, working_dir, scale=True):
"""
Get aspect at dtm_l2 resolution
:param dtm_l2:
:param working_dir:
:param scale:
:return:
"""
display_parameters(locals(), "get_aspect")
# running gdaldem to get aspect
logger.debug("gdal dem")
aspect_raw = os.path.join(working_dir,
getBasename(dtm_l2) + "_aspect_raw.tif")
logger.debug("slope file : " + aspect_raw)
commandGDAL = "gdaldem aspect " + dtm_l2 + " "
if scale:
commandGDAL += " -compute_edges "
commandGDAL += aspect_raw
logger.debug("command : " + commandGDAL)
os.system(commandGDAL)
logger.debug("gdal dem ok")
aspect_rad = os.path.join(working_dir,
getBasename(dtm_l2) + "_aspect_rad.tif")
# l'orientation est exprimée en degre float 32, on remet en radians * 100 codé sur 16 bits
# les degres vont de 0 à 360, on remet entre 0 et 2pi*100
cmd = "gdal_translate -ot Int16 -scale 0 360 0 628 " + aspect_raw + " " + aspect_rad
os.system(cmd)
# Necessary step ?
seuil = -500
output_aspect = os.path.join(working_dir,
getBasename(dtm_l2) + "_aspect.tif")
bandmath([aspect_rad], "(im1b1<" + str(seuil) + "?0:im1b1)", output_aspect)
logger.info("Output aspect")
logger.info("\t aspect : " + output_aspect)
return output_aspect
def hdr_creation(input_dir, output_filename, sensor):
display_parameters(locals(), "hdr_creation")
idMnt = extractSiteNicknameFromMntDirname(input_dir, sensor)
imageMnt = ""
if sensor == "l8":
count = "7"
files_order = ["SLP", "ASP", "ALT", "ALC", "MSK", "ASC", "SLC"]
mission = "LANDSAT_8"
applicableSite = "L8_TEST_MPL_SITDEF_S_"
imageMNT = searchOneFile(input_dir, "*ALT.TIF")
elif sensor == "s2":
count = "10"
files_order = [
"SLP_R1", "SLP_R2", "ASP_R1", "ASP_R2", "ALT_R1", "ALT_R2", "ALC",
"MSK", "ASC", "SLC"
]
mission = "SENTINEL-2_"
applicableSite = "S2__VM01_MPL_SITDEF_S_" + idMnt
imageMNT = searchOneFile(input_dir, "*ALT_R1.TIF")
info = gdalinfoO(imageMNT)
basename_out = os.path.basename(os.path.splitext(output_filename)[0])
logging.debug(basename_out)
xml_file = os.path.join(os.path.dirname(__file__), "xml_template.xml")
tree = ET.parse(xml_file)
# ET.register_namespace("", "http://eop-cfi.esa.int/CFI")
root = getRoot()
#tree.getroot()
# root = ET.Element("Earth_Explorer_Header", schema_version = "1.00", xmlns = "http://eop-cfi.esa.int/CFI", xsi = "http://www.w3.org/2001/XMLSchema-instance", type = "REFDE2_Header_Type", schemaLocation = "http://eop-cfi.esa.int/CFI AUX_REFDE2_ReferenceDemDataLevel2.xsd")
a1 = ET.SubElement(root, "Fixed_Header")
toto = ET.SubElement(a1, "File_Name")
toto.text = basename_out
toto = ET.SubElement(a1, "File_Description")
toto.text = "ReferenceDemDataLevel2"
toto = ET.SubElement(a1, "Notes")
toto.text = "String"
toto = ET.SubElement(a1, "Mission")
toto.text = mission
toto = ET.SubElement(a1, "File_Class")
toto.text = "TEST"
toto = ET.SubElement(a1, "File_Type")
toto.text = "AUX_REFDE2"
b1 = ET.SubElement(a1, "Validity_Period")
toto = ET.SubElement(b1, "Validity_Start")
toto.text = "UTC=2006-07-01T18:11:45"
toto = ET.SubElement(b1, "Validity_Stop")
toto.text = "UTC=9999-99-99T99:99:99"
toto = ET.SubElement(a1, "File_Version")
toto.text = "0001"
b2 = ET.SubElement(a1, "Source")
toto = ET.SubElement(b2, "System")
toto.text = "MACCS"
toto = ET.SubElement(b2, "Creator")
toto.text = "Earth Explorer CFI"
toto = ET.SubElement(b2, "Creator_Version")
toto.text = "1.1"
toto = ET.SubElement(b2, "Creation_Date")
toto.text = "UTC=2010-02-10T11:00:00"
a = ET.SubElement(root, "Variable_Header")
b2 = ET.SubElement(a, "Main_Product_Header")
ET.SubElement(b2, "List_of_Consumers", count="0")
ET.SubElement(b2, "List_of_Extensions", count="0")
b3 = ET.SubElement(a, "Specific_Product_Header")
b4 = ET.SubElement(b3, "Instance_Id")
toto = ET.SubElement(b4, "Applicable_Site_Nick_Name")
toto.text = idMnt
toto = ET.SubElement(b4, "File_Version")
toto.text = (os.path.basename(input_dir).split("_")[-1]).split(".")[0]
b5 = ET.SubElement(b3, "List_of_Applicable_SiteDefinition_Ids", count="1")
toto = ET.SubElement(b5, "Applicable_SiteDefinition_Id", sn="1")
toto.text = applicableSite
b6 = ET.SubElement(b3, "DEM_Information")
b7 = ET.SubElement(b6, "Cartographic")
b8 = ET.SubElement(b7, "Coordinate_Reference_System")
toto = ET.SubElement(b8, "Code")
toto.text = "EPSG:{}".format(info.getIntEpsgCode())
toto = ET.SubElement(b8, "Short_Description")
toto.text = ""
b8 = ET.SubElement(b7, "Upper_Left_Corner")
toto = ET.SubElement(b8, "X", unit="m")
toto.text = str(info.getOrigin()[0])
toto = ET.SubElement(b8, "Y", unit="m")
toto.text = str(info.getOrigin()[0])
b8 = ET.SubElement(b7, "Sampling_Interval")
toto = ET.SubElement(b8, "By_Line", unit="m")
toto.text = str(info.getPixelSize()[0])
toto = ET.SubElement(b8, "By_Column", unit="m")
toto.text = str(info.getPixelSize()[1])
b8 = ET.SubElement(b7, "Size")
toto = ET.SubElement(b8, "Lines")
toto.text = str(info.getSize()[0])
toto = ET.SubElement(b8, "Columns")
toto.text = str(info.getSize()[0])
toto = ET.SubElement(b6, "Mean_Altitude_Over_L2_Coverage", unit="m")
toto.text = "10"
toto = ET.SubElement(b6,
"Altitude_Standard_Deviation_Over_L2_Coverage",
unit="m")
toto.text = "20"
toto = ET.SubElement(b6, "Nodata_Value")
toto.text = "NIL=N/A"
toto = ET.SubElement(b6, "L2_To_DEM_Subsampling_Ratio")
toto.text = "1"
toto = ET.SubElement(b6, "Comment")
toto.text = ""
# b = ET.SubElement(a, "Specific_Product_Header")
c = ET.SubElement(b3, "DBL_Organization")
d = ET.SubElement(c, "List_of_Packaged_DBL_Files", count=count)
for image in xrange(len(files_order)):
paths_to_file = glob.glob(
os.path.join(input_dir, "*" + files_order[image] + "*"))
if paths_to_file:
path_to_file = paths_to_file[0]
logging.debug(path_to_file)
e1 = ET.SubElement(d, "Packaged_DBL_File", sn=str(image + 1))
ET.SubElement(e1, "Relative_File_Path").text = os.path.join(
os.path.basename(os.path.normpath(input_dir)),
os.path.basename(path_to_file))
ET.SubElement(e1, "File_Definition")
# tree = ET.ElementTree(root)
# output_file = os.sys.argv[1]
logging.debug("############################")
logging.debug(output_filename)
# tree.write(sys.argv[2], encoding = "utf-8", xml_declaration = True, method = "xml")
# xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent = " ")
# with open(output_filename, "w") as f:
# f.write(xmlstr.encode('utf-8'))
tree = ET.ElementTree(root)
f = open(output_filename, "w")
f.write(
ET.tostring(tree,
pretty_print=True,
xml_declaration=True,
encoding="UTF-8"))
f.close()
def generate_wb(dtm_l2_coarse, working_dir):
"""
Behaviour for downloaded tiles from DownloadSRTMTiles application.
It runs the following steps:
-unzip all downloaded tiles
-creates a vrt with all hgt files
-crop the vrt with the given image as reference
"""
display_parameters(locals(), "get_wb")
working_dir_wb = os.path.join(working_dir, "working_dir_wb")
if os.path.isdir(working_dir_wb):
#empty wd directory
[os.remove(x) for x in glob.glob(os.path.join(working_dir_wb, "*"))]
else:
os.mkdir(working_dir_wb)
# download wb tiles
logger.debug("Downloading srtm wb tiles")
# DownloadSWBDTiles_dl(data, working_dir_wb)
DownloadSWBDTiles_dl(dtm_l2_coarse, working_dir_wb)
logger.debug("Srtm wb tiles ok")
logger.debug("Unziping srtm tiles")
epsg_code, a_x, a_y, b_x, b_y, _, _, _, _ = get_image_info_with_gdal(
dtm_l2_coarse)
unzip_files(working_dir_wb)
logger.debug("Unzip ok")
logger.debug("Reproject DTM coarse in wgs84")
# set a projection to all shapefiles and rasterize on dtm at L2_Coarse resolution
logger.debug("Assigning projection to wb")
listWaterBodies = []
for shapefile in glob.glob(os.path.join(working_dir_wb, "*.shp")):
# set projection to wb
projfile = os.path.splitext(shapefile)[0] + ".prj"
logger.debug("projfile :" + projfile)
my_proj = open(projfile, 'w')
my_proj.write(
'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]'
)
my_proj.close()
logger.debug("rasterizing wb")
shape_temp = os.path.splitext(
shapefile)[0] + "_reproject" + os.path.splitext(shapefile)[1]
ogr_command = "ogr2ogr -t_srs EPSG:" + str(
epsg_code) + " -s_srs EPSG:4326 " + shape_temp + " " + shapefile
os.system(ogr_command)
shape_clipped = clipVectorWithOGR_Extent(shape_temp,
[a_x, a_y, b_x, b_y],
working_dir_wb)
temp = rasterizationWithSupportImage(shape_clipped, dtm_l2_coarse,
working_dir_wb)
logger.debug("rasterizing wb ok")
listWaterBodies.append(temp)
logger.debug("Projection to wb ok")
logger.debug(listWaterBodies)
wb = os.path.join(working_dir, getBasename(dtm_l2_coarse) + "_MSK.TIF")
logger.debug("OR between images")
if not listWaterBodies == []:
formula = "im1b1 "
for i in range(1, len(listWaterBodies)):
formula += 'or im' + str(i + 1) + "b1 "
bandmath(listWaterBodies, formula, wb, "uint8")
logger.info("Output WB")
logger.info("\t WB : " + wb)
return wb
def l8_align(input_dir, l8_vector, output_directory_path_row_data, working_dir, path=0, row=0):
"""
Runs the crop of all input data
# crop all input tiles to match l8 real extent
:param input_dir:
:param l8_vector:
:param output_data_directory:
:param working_dir:
:param path:
:param row:
:return:
"""
display_parameters(locals(), "l8_align")
time_start = time.time()
extent_wgs84_vector, extent_wgs84, extent_feature = get_extent_from_l8_extent(l8_vector, path, row, working_dir, "4326")
#creates the output data directory containing cropped images (5)
output_product_dir_data = os.path.join(output_directory_path_row_data,
os.path.basename(os.path.normpath(input_dir)))
logging.debug("ouput_product_dir_data {}".format(output_product_dir_data))
if not os.path.isdir(output_product_dir_data):
os.makedirs(output_product_dir_data)
image = searchOneFile(input_dir, "*_B1.TIF")
if not image:
print "B1 is missing"
sys.exit(2)
logger.debug(image)
# txt file
logging.debug(glob.glob(os.path.join(input_dir, "*_MTL.txt")))
txt_file = glob.glob(os.path.join(input_dir, "*_MTL.txt"))[0]
logger.debug(txt_file)
epsg_code = get_image_info_with_gdal(image, get_only_epsg = True)
# print epsg_code
logging.debug("epsg code{}".format(epsg_code))
shape_temp = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_reprojected" + os.path.splitext(os.path.basename(l8_vector))[1])
logger.debug(shape_temp)
#ogr_command = "ogr2ogr -lco \"OGR_ENABLE_PARTIAL_REPROJECTION\"=TRUE -t_srs EPSG:" + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector #
ogr_command = 'ogr2ogr -where \'"PATH"=' + str(path) + ' and "ROW"=' + str(row) + '\' -t_srs EPSG:' + str(epsg_code) + " -s_srs EPSG:4326" " " + shape_temp + " " + l8_vector
logging.debug(ogr_command)
os.system(ogr_command)
#shape_env = os.path.join(working_dir, os.path.splitext(os.path.basename(l8_vector))[0] + "_env" + os.path.splitext(os.path.basename(l8_vector))[1])
shape_env, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code))
update_txt(txt_file, shape_env_points, output_product_dir_data, epsg_code)
bqa_temp = ""
bqd_out = ""
for tif_file in glob.glob(os.path.join(input_dir, "*.TIF")):
data_out = os.path.join(output_product_dir_data, os.path.basename(tif_file))
resolution = gdalinfoO(tif_file).getPixelSize()[0]
scale_factor = float(config.L2_coarse_resolution)
extent, shape_env_points = get_feature_enveloppe(shape_temp, working_dir, int(epsg_code), scale_factor)
if "BQA" in tif_file:
bqa_temp = os.path.join(working_dir, os.path.basename(tif_file))
bqd_out = data_out
crop_with_extent(tif_file, shape_env_points, bqa_temp)
else:
crop_with_extent(tif_file, shape_env_points, data_out)
# crop_with_mask(tif_file, extent, data_out)
if os.path.isfile(bqa_temp):
image_support = searchOneFile(output_product_dir_data, "*_B1.TIF")
if not image_support:
print "B1 is missing in output directory"
sys.exit(2)
bandmath([image_support, bqa_temp], "(im1b1==0&&im2b1==0?0:im2b1)", bqd_out)
else:
logging.info("{} already processed.".format(output_product_dir_data))
interval = time.time() - time_start
logger.info('Total time in seconds:' + str(interval))
return output_product_dir_data
def get_arguments():
"""
Manages inputs
:return:
"""
# ####### MANAGING ARGUMENTS ########
# TODO : set path to downloadwb as an argument
# check inputs
logger.info("######## MANAGING ARGUMENTS ########")
logger.debug("Checking arguments")
argParser = argparse.ArgumentParser()
required_arguments = argParser.add_argument_group('required arguments')
required_arguments.add_argument('-i', '--input_directory', required=True,
help='Path to input image')
required_arguments.add_argument('-o', '--output_directory', required=True,
help='Path to output directory')
required_arguments.add_argument('-v', '--l8_vector', required=True,
help='Path to l8 vector ex: wrs2_descending/wrs2_descending.shp')
required_arguments.add_argument('-w', '--working_directory', required=True,
help='Path to working directory')
required_arguments.add_argument('-t', '--tile_id', required=True,
help='Current product tile id')
logger.debug(len(sys.argv))
if len(sys.argv) != 11:
usage(argParser)
args = argParser.parse_args(sys.argv[1:])
input_directory = os.path.realpath(args.input_directory)
output_directory = os.path.realpath(args.output_directory)
l8_vector = args.l8_vector
working_dir = args.working_directory
tile_id = args.tile_id
logger.debug("Arguments ok")
# crating directories if they don't exist
logger.debug("Managing output directories")
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
if not os.path.isdir(working_dir):
os.mkdir(working_dir)
if not os.path.isdir(input_directory):
print "Error, input dir is missing "
usage(argParser, 2)
logger.debug("Output directories ok")
# checking if inputs exist
if not input_directory:
print "input image missing"
sys.exit(2)
if not l8_vector:
print "l8_vector missing"
sys.exit(2)
# get path, row of the product
txt = os.path.join(input_directory, tile_id + "_MTL.txt")
path, row = getPathRowFromMTL(txt)
display_parameters(locals(), "l8_align_get_arguments")
return input_directory, l8_vector, output_directory, working_dir, path, row