def publish_data_GriddedLivestock(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
product = "Gridded Livestock of the World v. 2.01"
sldname = "ghg_"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0].capitalize() + " " + info[1] + " - " + info[2]
sldname += get_filename(input_file).lower()
date = info[2] + "01"
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def publish_data_GriddedLivestock(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "Gridded Livestock of the World v. 2.01"
sldname = "ghg_"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0].capitalize() + " " + info[1] + " - " + info[2]
sldname += get_filename(input_file).lower()
date = info[2] + "01"
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def publish_data_GriddedLivestock(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "Gridded Livestock of the World v. 2.01"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0].capitalize() + " " + info[1] + " - " + info[2]
sldname = "ghg_" + get_filename(input_file).lower() + "_EN"
date = info[2]
metadata_def = create_metadata(title, product, date, sldname)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
def process(input_folder, output_folder, mask_file):
# create the folder
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.mkdir(output_folder)
try:
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(
input_file) + ".tif"
#gdal_calc.py -A wheat_seasonal_act_biomprod_doukkala_mask.tif -B asd --outfile=processed.tif --calc="B*A" --NoDataValue=0
#gdal_calc.py -A ndvi_mask.tif -B --outfile=result.tif --calc="B*(A>0)" --NoDataValue=-3000
cmd = 'gdal_calc.py -A ' + input_file + ' -B ' + mask_file + ' --outfile=' + output_filename + ' --calc="B*A" --NoDataValue=-3000'
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
output, error = process.communicate()
print output
print error
except Exception, e:
print e
pass
def process(input_folder, output_folder):
# create the folder
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.mkdir(output_folder)
try:
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(
input_file) + ".tif"
#cmd = 'gdalwarp -srcnodata 0 -dstnodata -3000 -te -1010580.0921138877747580 3810893.3240307914093137 -894408.7707638647407293 3894810.6912959185428917 ' + input_file + ' ' + output_filename
cmd = 'gdalwarp -srcnodata -3000 -dstnodata -3000 -te -1010580.0921138877747580 3810893.3240307914093137 -894408.7707638647407293 3894810.6912959185428917 ' + input_file + ' ' + output_filename
print cmd
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
output, error = process.communicate()
print output
print error
except Exception, e:
print e
pass
def process(input_folder, output_folder, srcproj, dstproj):
print "Processing data %s %s %s ", input_folder, output_folder
ext = "shp"
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.makedirs(output_folder)
try:
input_files = glob.glob(input_folder + "/*." + ext)
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(
input_file) + "." + ext
cmd = "ogr2ogr -f 'ESRI Shapefile' -s_srs " + srcproj + " -t_srs " + dstproj + " " + output_filename + " " + input_file
print cmd
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
output, error = process.communicate()
print output
print error
except Exception, e:
print e
pass
def publish_data_wheat(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
if "water_productivity" in input_file:
name = info[0] + " " + info[1] + " " + info[2]
sldname = info[1] + "_" + info[2]
metadata_def = create_metadata(name, sldname, "201410", info[3])
manager.publish_coverage(input_file, metadata_def)
if "yieldgap" in input_file:
name = info[0] + " " + info[1]
sldname = info[1]
metadata_def = create_metadata(name, sldname, "201410", info[3])
manager.publish_coverage(input_file, metadata_def)
if "yield" in input_file:
name = info[0] + " " + info[1] + " " + info[2]
sldname = info[2]
metadata_def = create_metadata(name, sldname, "201410", info[3])
manager.publish_coverage(input_file, metadata_def)
if "seasonal" in input_file:
name = info[0] + " " + info[1] + " " + info[2] + " " + info[3]
sldname = info[3]
metadata_def = create_metadata(name, sldname, "201410", info[4])
manager.publish_coverage(input_file, metadata_def)
def import_data_earthstat(input_folder):
dir = glob.glob(input_folder + "*")
for d in dir:
if os.path.isdir(d):
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
import_data(input_file, "es10_10", get_filename(input_file))
def import_data_earthstat(input_folder):
dir = glob.glob(input_folder + "*")
for d in dir:
if os.path.isdir(d):
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
import_data(input_file, "es10_10", get_filename(input_file))
def publish_burnerdareas():
path = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/GFED4_BURNEDAREAS_BY_LANDCOVER/"
input_dir = glob.glob(path + "*")
for d in input_dir:
input_files = glob.glob(d + "/*.tiff")
# sld and workspace
sldname = "ghg_burnedareas" + "_EN"
workspace = "fenix:"
for input_file in input_files:
#print input_file
if "3857" in input_file:
if "humid" in input_file.lower() or "allforests" in input_file.lower():
print input_file
info = str.split(get_filename(input_file), "_")
date = info[3]
filename = get_filename(input_file).rsplit('_', 1)[0]
uid = workspace + get_filename(filename).lower()
product = burned_areas_switch(filename)
title = product + " " + info[3]
metadata_def = create_metadata(title, product, date, sldname)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
#manager.publish_coverage(input_file, metadata_def, False, False)
else:
info = str.split(get_filename(input_file), "_")
date = info[4]
filename = get_filename(input_file).rsplit('_', 1)[0]
uid = workspace + get_filename(filename).lower()
product = burned_areas_switch(input_file)
title = product + " " + info[4]
metadata_def = create_metadata(title, product, date, sldname)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
#manager.publish_coverage(input_file, metadata_def, False, False)
else: #4326
info = str.split(get_filename(input_file), "_")
if len(info) >= 5:
title = info[0] + " " + info[1] + " " + info[2] + " " + info[3] + " " + info[4] + " - 4326"
date = info[4]
else:
title = info[0] + " " + info[1] + " " + info[2] + " " + info[3]
date = info[3]
product = get_filename(input_file).replace('_', ' ') + " (4326)"
metadata_def = create_metadata(title, product, date, sldname)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
def process():
if os.path.isdir(output_folder):
log.info("already exists")
else:
os.mkdir(output_folder)
output_file = output_folder + get_filename(input_file) + ".tif"
process_layers(input_file, output_file)
print output_file
def publish_data_winter_crop_classification(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
product = "Doukkala - wheat seasonal"
title = "Winter Crop Classification"
sldname = "morocco_winter_crop_classification"
date = "201401" # FAKE DATE!!!
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
print metadata_def
def publish_data_winter_crop_classification(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
product = "Doukkala - wheat seasonal"
title = "Winter Crop Classification"
sldname = "morocco_winter_crop_classification"
date = "201401" # FAKE DATE!!!
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
print metadata_def
def publish_data_temperature(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = "Temperature " + info[2] + " - " + info[3].capitalize()
sldname = "morocco_temperature"
date = info[2]
product = info[3].capitalize() + " - Temperature"
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
print metadata_def
def publish_data_temperature(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = "Temperature " + info[2] + " - " + info[3].capitalize()
sldname = "morocco_temperature"
date = info[2]
product = info[3].capitalize() + " - Temperature"
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
print metadata_def
def process_file(input_file, output_folder, process_layer_parameters):
try:
output_filename = output_folder + "/" + get_filename(input_file) + ".tif"
print(output_filename)
# create a geotiff + overviews
process_layers(input_file, output_filename, process_layer_parameters)
except Exception, e:
print e
pass
def process_earthstat():
if os.path.isdir(output_folder):
log.info("already exists")
else:
os.mkdir(output_folder)
dir = glob.glob(input_folder + "*")
for d in dir:
if os.path.isdir(d):
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
output_file = output_folder + get_filename(input_file) + ".tif"
process_layers(input_file, output_file)
print output_file
def process(self):
# process zip file
path = unzip(self.xls_parser["file_path"])
files_path = glob.glob(os.path.join(path, "*"))
data = []
for file_path in files_path:
print file_path
self.xls_parser["file_path"] = file_path
d = {}
d["name"], d["year"] = self.parse_filename(get_filename(file_path))
d["data"] = self.process_xls()
data.append(d)
return data
def publish_data_modis_landcover(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "MODIS - Land Cover Type UMD"
sldname = "modis_land_cover"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1] + " " + info[2] + " " + info[3] + " " + info[4] + " " + info[5] + " - " + info[6]
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
date = info[6] + info[7]
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def process(self):
# process zip file
path = unzip(self.xls_parser["file_path"])
files_path = glob.glob(os.path.join(path, "*"))
data = []
for file_path in files_path:
print file_path
self.xls_parser["file_path"] = file_path
d = {}
d["name"], d["year"] = self.parse_filename(get_filename(file_path))
d["data"] = self.process_xls()
data.append(d)
return data
def process_earthstat():
if os.path.isdir(output_folder):
log.info("already exists")
else:
os.mkdir(output_folder)
dir = glob.glob(input_folder + "*")
for d in dir:
if os.path.isdir(d):
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
output_file = output_folder + get_filename(input_file) + ".tif"
process_layers(input_file, output_file)
print output_file
def publish_data_Climate_Zones_processed(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
product = "JRC climate zone"
sldname = "ghg_jrc_climate_zone_0.25deg"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1].lower() + " - " + info[2].lower()
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
date = None
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def publish_data_Climate_Zones_processed(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "JRC climate zone"
sldname = "ghg_jrc_climate_zone_0.25deg"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1].lower() + " - " + info[2].lower()
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
date = None
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def publish_layers():
print "Publish Layers"
files = glob.glob(output_folder + "/*.tif")
print files
for f in files:
print f
# read filename
name = get_filename(f)
# get commodity name
title = name.replace("_", " ").replace("1", "").capitalize()
print title
# create metadata
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
# TODO: in theory 1970-2010?
# metadata_def["meContent"]["seCoverage"]["coverageTime"]["from"] = "1970"
# metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = "2010"
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"][
"codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{
"code":
"EARTHSTAT"
}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{
"code":
"EARTHSTAT"
}]
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"][
"name"] = "earthstat_area"
# merging metata to raster metadata
metadata_def = merge_layer_metadata("raster", metadata_def)
print metadata_def
# get type
if "area" in title:
print title
manager.publish_coverage(f, metadata_def)
def publish_data_modis_landcover(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
product = "MODIS - Land Cover Type UMD"
sldname = "modis_land_cover"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1] + " " + info[2] + " " + info[
3] + " " + info[4] + " " + info[5] + " - " + info[6]
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
date = info[6] + info[7]
metadata_def = create_metadata(title, product, sldname, date, None)
print metadata_def
def publish_data(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
if "irrigated" not in info[0]:
pass
# metadata_def = create_metadata(info[0], info[1], info[2], info[3])
# manager.publish_coverage(input_file, metadata_def)
else:
name = info[0] + " " + info[1] + " " + info[2]
sldname = info[0] + "_" + info[1] + "_" + info[2]
metadata_def = create_metadata(name, sldname, "201410", info[3])
manager.publish_coverage(input_file, metadata_def)
def publish_data_timeseries(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "ndvi" in input_file:
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = info[0].upper() + " " + info[1] + " - " + info[2].capitalize()
product = info[2].capitalize() + " - " + info[0].upper()
if "precipitation" in input_file:
title = info[0].capitalize() + " " + info[1] + " - " + info[2].capitalize()
product = info[2].capitalize() + " - " + info[0].capitalize()
sldname = "morocco_" + info[0]
date = info[1]
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
print metadata_def
def publish_data_wheat_seasonal_irrigated(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
product = "Doukkala - wheat seasonal"
sldname = "morocco_"
date = "201401" # FAKE DATE!!!
if "water" in input_file:
title = info[0] + " " + info[2] + " " + info[3]
sldname += "water_productivity"
if "yieldgap" in input_file:
title = info[0] + " " + info[2]
sldname += "yieldgap"
if "yield" in input_file:
title = info[0] + " " + info[2] + " " + info[3]
sldname += "yield"
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
print metadata_def
def publish_data_wheat_seasonal_irrigated(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
product = "Doukkala - wheat seasonal"
sldname = "morocco_"
date = "201401" # FAKE DATE!!!
if "water" in input_file:
title = info[0] + " " + info[2] + " " + info[3]
sldname += "water_productivity"
if "yieldgap" in input_file:
title = info[0] + " " + info[2]
sldname += "yieldgap"
if "yield" in input_file:
title = info[0] + " " + info[2] + " " + info[3]
sldname += "yield"
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
print metadata_def
def publish_layers():
print "Publish Layers"
files = glob.glob(output_folder + "/*.tif")
print files
for f in files:
print f
# read filename
name = get_filename(f)
# get commodity name
title = name.replace("_", " ").replace("1", "").capitalize()
print title
# create metadata
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
# TODO: in theory 1970-2010?
# metadata_def["meContent"]["seCoverage"]["coverageTime"]["from"] = "1970"
# metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = "2010"
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "EARTHSTAT"}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "EARTHSTAT"}]
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = "earthstat_area"
# merging metata to raster metadata
metadata_def = merge_layer_metadata("raster", metadata_def)
print metadata_def
# get type
if "area" in title:
print title
manager.publish_coverage(f, metadata_def)
def publish_data_Climate_Zones_processed(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "JRC climate zone"
sldname = "ghg_jrc_climate_zone_0.25deg" + "_EN"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1].lower() + " - " + info[2].lower()
map_projection_code = "EPSG:3857"
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
map_projection_code = "EPSG:4326"
date = "2010"
metadata_def = create_metadata(title, product, date, sldname, map_projection_code)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
def publish_data_modis_landcover(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
product = "MODIS - Land Cover Type UMD"
sldname = "modis_land_cover" + "_EN"
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
title = info[0] + " " + info[1] + " " + info[2] + " " + info[3] + " " + info[4] + " " + info[5] + " - " + info[6]
map_projection_code = "EPSG:3857"
if "4326" in info[4]:
title += " (4326)"
product += " (4326)"
map_projection_code = "EPSG:4326"
date = info[6]
metadata_def = create_metadata(title, product, date, sldname, map_projection_code)
try:
print json.dumps(metadata_def)
data_manager.publish_coveragestore(input_file, metadata_def, False, False, True)
except Exception, e:
print e
def publish_data_meteo_evapotranspiration(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "actual" in input_file:
type = "actual"
if "potential" in input_file:
type = "potential"
if "ETref" in input_file:
type = "reference"
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = type.capitalize() + " evapotransipiration " + " " + info[1] + " - " + info[2].capitalize()
sldname = "morocco_evapotransipiration"
date = info[1]
product = info[2].capitalize() + " - " + type + " evapotransipiration"
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
def process(input_folder, output_folder, mask_file):
# create the folder
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.mkdir(output_folder)
try:
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
output_filename = output_folder + "/wheat_" + get_filename(input_file) + ".tif"
#gdal_calc.py -A wheat_seasonal_act_biomprod_doukkala_mask.tif -B asd --outfile=processed.tif --calc="B*A" --NoDataValue=0
cmd = 'gdal_calc.py -A ' + input_file + ' -B ' + mask_file + ' --outfile=' + output_filename + ' --calc="B*A" --NoDataValue=0'
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
except Exception, e:
print e
pass
def harvest_folder(path):
"""
Harvest files in a path
:param path:
:return:
"""
manager = Manager(settings)
types = ('*.tiff', '*.geotiff', "*.gtiff", "*.tif")
# for each tiff, geotiff, tif, gtiff
files_grabbed = []
for file_type in types:
files_grabbed.extend(glob.glob(os.path.join(path, file_type)))
for file_path in files_grabbed:
path, filename, name = filesystem.get_filename(file_path, True)
#log.info("%s %s %s " % (path, filename, name))
default_workspace = manager.geoserver.get_default_workspace()
name = sanitize_name(name)
if manager.geoserver.check_if_layer_exist(name, default_workspace) is False:
metadata_file = os.path.join(path, name + ".json")
log.info("Check Metadata File: %s" % metadata_file)
if os.path.isfile(metadata_file):
metadata_json = json.loads(open(metadata_file).read())
log.info(metadata_json)
else:
log.warn("Metadata '%s' doesn't exists " % metadata_file)
log.warn("Setting the default metadata file for '%s' " % name)
# TODO: create a default metadata if the file doesn't exists it doesn't exists
metadata = deepcopy(default_metadata_json)
log.info("Prossessing the raster '%s' " % (name))
metadata = merge_layer_metadata(raster_template, metadata_json)
# process metadata with the default workspace if uid is not set in the metadata
if "uid" not in metadata:
metadata["uid"] = default_workspace + ":" + name
manager.publish_coverage(file_path, metadata)
log.info("Raster published '%s' " % (name))
else:
log.warn("Coverage '%s' already exists " % name)
def publish_data_timeseries(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "ndvi" in input_file:
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = info[0].upper(
) + " " + info[1] + " - " + info[2].capitalize()
product = info[2].capitalize() + " - " + info[0].upper()
if "precipitation" in input_file:
title = info[0].capitalize(
) + " " + info[1] + " - " + info[2].capitalize()
product = info[2].capitalize() + " - " + info[0].capitalize()
sldname = "morocco_" + info[0]
date = info[1]
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
print metadata_def
def publish_data_meteo_evapotranspiration(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "actual" in input_file:
type = "actual"
if "potential" in input_file:
type = "potential"
if "ETref" in input_file:
type = "reference"
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
title = type.capitalize() + " evapotransipiration " + " " + info[
1] + " - " + info[2].capitalize()
sldname = "morocco_evapotransipiration"
date = info[1]
product = info[2].capitalize() + " - " + type + " evapotransipiration"
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
def process_landcover():
path = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/GFED4_BURNEDAREAS_BY_LANDCOVER/"
input_dir = glob.glob(path + "*")
print input_dir
for d in input_dir:
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
print d
print input_file
filename = get_filename(input_file)
source_path = [input_file]
print filename
output_path = d + "/"
output_file_name = filename + "_3857"
print output_path
for obj in objs:
obj["source_path"] = source_path
obj["output_path"] = output_path
obj["output_file_name"] = output_file_name
print obj
source_path = process_step(obj)
def process(input_folder, output_folder, process_layer_parameters):
print "Processing data %s %s %s ", input_folder, output_folder, process_layer_parameters
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.makedirs(output_folder)
try:
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
print input_file
output_filename = output_folder + "/" + get_filename(input_file) + ".tif"
print(output_filename)
# create a geotiff + overviews
process_layers(input_file, output_filename, process_layer_parameters)
except Exception, e:
print e
pass
def process_landcover():
path = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/GFED4_BURNEDAREAS_BY_LANDCOVER/"
input_dir = glob.glob(path + "*")
print input_dir
for d in input_dir:
input_files = glob.glob(d + "/*.tif")
for input_file in input_files:
print d
print input_file
filename = get_filename(input_file)
source_path = [input_file]
print filename
output_path = d + "/"
output_file_name = filename + "_3857"
print output_path
for obj in objs:
obj["source_path"] = source_path
obj["output_path"] = output_path
obj["output_file_name"] = output_file_name
print obj
source_path = process_step(obj)
def publish_data_wheat_seasonal(input_folder):
input_files = glob.glob(input_folder + "/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "wheat_productivity" not in input_file and "yieldgap" not in input_file:
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
#title = info[4].capitalize() + " - " + info[0] + " " + info[1] + " " + info[2] + " - " + info[3]
# title = info[4].capitalize() + " - " + info[0] + " " + info[1] + " " + info[2] + " - " + info[3]
title = info[2] + " - " + info[3]
sldname = "morocco_" + info[2]
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
#print metadata_def
#manager.publish_coverage(input_file, metadata_def)
if "water" in input_file:
title = info[2] + " " + info[3]
sldname = "morocco_" + info[2] + "_" + info[3]
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
#print metadata_def
if "yieldgap" in input_file:
title = info[2]
sldname = "morocco_" + info[2]
date = "201401" # FAKE DATE!!!
product = info[3].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
if "evapotransipiration" in input_file:
title = info[2] + " - " + info[3]
sldname = "morocco_evapotransipiration_wheat"
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date,
get_measurement_unit(title))
# publishing
print metadata_def
def process(input_folder, output_folder):
# create the folder
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.mkdir(output_folder)
try:
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(input_file) + ".tif"
#cmd = 'gdalwarp -srcnodata 0 -dstnodata -3000 -te -1010580.0921138877747580 3810893.3240307914093137 -894408.7707638647407293 3894810.6912959185428917 ' + input_file + ' ' + output_filename
cmd = 'gdalwarp -srcnodata -3000 -dstnodata -3000 -te -1010580.0921138877747580 3810893.3240307914093137 -894408.7707638647407293 3894810.6912959185428917 ' + input_file + ' ' + output_filename
print cmd
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
print output
print error
except Exception, e:
print e
pass
def calc():
# take folders
folders = glob.glob(input_folder)
for folder in folders:
print input_folder
# create output folder /output
# covert to geotiff 3857 the file
output_folder = folder + "/output"
if os.path.isdir(output_folder):
remove(output_folder + "/*.tif")
else:
os.mkdir(output_folder)
input_files = glob.glob(folder +"/*.tif")
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(input_file) + ".tif"
print(output_filename)
# create a geotiff + overviews
process_layers(input_file, output_filename)
def calc_trmm():
# take folders
folders = glob.glob(input_folder)
for folder in folders:
print input_folder
# create output folder /output
# covert to geotiff 3857 the file
output_folder = folder + "/output"
if os.path.isdir(output_folder):
remove(output_folder + "/*.tif")
else:
os.mkdir(output_folder)
input_files = glob.glob(folder + "/*.tif")
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(
input_file) + ".tif"
print(output_filename)
# create a geotiff + overviews
process_layers(input_file, output_filename)
def process(input_folder, output_folder, srcproj, dstproj):
print "Processing data %s %s %s ", input_folder, output_folder
ext = "shp"
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.makedirs(output_folder)
try:
input_files = glob.glob(input_folder + "/*." + ext)
for input_file in input_files:
output_filename = output_folder + "/" + get_filename(input_file) + "." + ext
cmd = "ogr2ogr -f 'ESRI Shapefile' -s_srs " + srcproj + " -t_srs " + dstproj + " " + output_filename + " " + input_file
print cmd
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, error = process.communicate()
print output
print error
except Exception, e:
print e
pass
def publish_data_wheat_seasonal(input_folder):
input_files = glob.glob(input_folder +"/*.tif")
for input_file in input_files:
info = str.split(get_filename(input_file), "_")
if "wheat_productivity" not in input_file and "yieldgap" not in input_file:
#print "name %s, sld %s, date %s, area %s" % (info[0], info[1], info[2], info[3])
#title = info[4].capitalize() + " - " + info[0] + " " + info[1] + " " + info[2] + " - " + info[3]
# title = info[4].capitalize() + " - " + info[0] + " " + info[1] + " " + info[2] + " - " + info[3]
title = info[2] + " - " + info[3]
sldname = "morocco_" + info[2]
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
#print metadata_def
#manager.publish_coverage(input_file, metadata_def)
if "water" in input_file:
title = info[2] + " " + info[3]
sldname = "morocco_" + info[2] + "_" + info[3]
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
#print metadata_def
if "yieldgap" in input_file:
title = info[2]
sldname = "morocco_" + info[2]
date = "201401" # FAKE DATE!!!
product = info[3].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
if "evapotransipiration" in input_file:
title = info[2] + " - " + info[3]
sldname = "morocco_evapotransipiration_wheat"
date = "201401" # FAKE DATE!!!
product = info[4].capitalize() + " - " + info[0] + " " + info[1]
metadata_def = create_metadata(title, product, sldname, date, get_measurement_unit(title))
# publishing
print metadata_def
def publish_layers():
files = glob.glob(output_folder + "output/*.tif")
for f in files:
log.info("--------------------------------")
name = get_filename(f)
log.info("name %s %s" %(name, f))
month = int(name[5:7])
log.info("month %s" % month)
year = None
try:
year = int(name[8:12])
except Exception, e:
log.error("year error")
# TODO: what year for the DA?
if year is None:
year = "2014"
# get title name
title = name.replace("_", " ").capitalize()
# get metadata
from_date, to_date = get_range_dates_metadata(month, year)
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
# Sample of Metadata json
log.info("Creating metadata")
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"]["from"] = from_date
metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = to_date
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "TRMM"}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "TRMM"}]
# TODO: in theory should be the original file the onlineResource
metadata_def["meAccessibility"] = {}
metadata_def["meAccessibility"]["seDistribution"] = {}
metadata_def["meAccessibility"]["seDistribution"]["onlineResource"] = f
# TODO: added new field for the original resource (should we have two different metadata?)
#metadata_def["meAccessibility"]["seDistribution"]["originalResource"] = output_filename
# adding type of layer
aggregationProcessing = "none"
if "_avg" in name:
aggregationProcessing = "avg"
elif "_da" in name:
aggregationProcessing = "da"
metadata_def["meStatisticalProcessing"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"]["aggregationProcessing"] = aggregationProcessing;
# default style
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
if aggregationProcessing == "da":
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = "rainfall_" + aggregationProcessing
else:
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = "rainfall"
# merging metadata to the base raster one
metadata_def = merge_layer_metadata("raster", metadata_def)
# "seCoverage" : {
# "coverageTime" : {
# "from" : 1328405808080,
# "to": 1328405808080
# },
# "coverageGeographic" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "world"}]
# },
# "coverageSector" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "MODISQ1"}]
# }
# }]
log.info(metadata_def)
# publish layer
print manager.publish_coverage(f, metadata_def)
def publish_coveragestore(self, data, overwrite=False):
name = data["name"]
workspace = self.get_default_workspace() if "workspace" not in data else data["workspace"]
path = data["path"]
if workspace is not self.get_default_workspace():
if self.check_if_workspace_exist(workspace) is False:
raise PGeoException(errors[523]+": %s" % workspace)
if not overwrite:
if self.check_if_layer_exist(name, workspace):
raise PGeoException(errors[520]+": %s" % name)
# default geotiff headers and extension
headers = get_headers("tiff")
ext = "geotiff"
# check if the layer is a tfw (a zipfile)
#TODO: make it work with tif+tfw
path_folder_tif, filename_tif, name_tif = get_filename(path, True)
file_tfw = os.path.join(path_folder_tif, name_tif + ".tfw")
if os.path.isfile(file_tfw):
# handle 'tfw' (worldimage)
bundle = [path, file_tfw]
log.info(bundle)
archive = zip_files(name, bundle, path_folder_tif)
log.info(archive)
#message = open(archive, 'rb')
message = archive
headers = get_headers("zip")
ext = "worldimage"
elif isinstance(path, basestring):
message = open(path, 'rb')
else:
message = path
log.info(message)
try:
# Add layer to coveragestore
cs_url = url(self.service_url, ["workspaces", workspace, "coveragestores", name, "file." + ext])
self._publish_layer(cs_url, "PUT", message, headers, 201)
# Update metadata of the layer
headers = get_headers("json")
json_data = deepcopy(data)
del json_data['name']
# # TODO: REMOVE IT!!!!!!
# json_data['nativeCRS'] = 'PROJCS["World_Mollweide",GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Mollweide"],PARAMETER["False_Easting",0],PARAMETER["False_Northing",0],PARAMETER["Central_Meridian",0],UNIT["Meter",1],AUTHORITY["EPSG","54009"]]'
# json_data['srs'] = "EPSG:54009"
# set has default enabled the layer
if "enabled" not in json_data:
json_data["enabled"] = True
# json to send to geoserver
update_layer = {
"coverage" : json_data
}
cs_url = url(self.service_url, ["workspaces", workspace, "coveragestores", name, "coverages", name + ".json"])
self._publish_layer(cs_url, "PUT", json.dumps(update_layer), headers, 200)
# TODO: check why doesn't update the default style
if 'defaultStyle' in json_data:
if 'name' in json_data['defaultStyle']:
self.set_default_style(name, json_data['defaultStyle']['name'])
except PGeoException, e:
log.error(e.get_message())
raise PGeoException(e.get_message(), e.get_status_code())
def publish():
folders = glob.glob(input_folder)
for folder in folders:
files = glob.glob(folder +"/output/*.tif")
for f in files:
log.info("--------------------------------")
log.info(f)
if "avg" in f:
print "avg"
name = get_filename(f)
to_day = name[name.index("_")+1:name.rindex("_")]
year = 1990
print year
from_day = int(to_day) - 15
print from_day
from_date = calendar.timegm(day_of_the_year_to_date(to_day, year).timetuple())
to_date = calendar.timegm(day_of_the_year_to_date(from_day, year).timetuple())
#title = name.replace("", " ").upper()
title = "TRMM SADC Avg - " + to_day
else:
log.info("--------------------------------")
log.info(f)
name = get_filename(f)
d = name[name.index("_")+1:name.rindex("_")]
year = d[0:4]
to_day = name[name.rindex("_")+1:]
print d
print year
print to_day
from_day = int(to_day) - 15
print from_day
fromdate = day_of_the_year_to_date(to_day, year)
todate = day_of_the_year_to_date(from_day, year)
from_date = calendar.timegm(fromdate.timetuple())
to_date = calendar.timegm(todate.timetuple())
print from_date
print to_date
# get title name
#title = name.replace("_", " ").capitalize()
if "Anomaly" in f:
title = "TRMM SADC Anomaly - " + str(fromdate)
else:
title = "TRMM SADC - " + str(fromdate)
# get metadata
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
# Sample of Metadata json
log.info("Creating metadata")
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"]["from"] = from_date
metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = to_date
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "TRMM-SADC"}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : "TRMM-SADC"}]
# TODO: in theory should be the original file the onlineResource
metadata_def["meAccessibility"] = {}
metadata_def["meAccessibility"]["seDistribution"] = {}
metadata_def["meAccessibility"]["seDistribution"]["onlineResource"] = f
# TODO: added new field for the original resource (should we have two different metadata?)
#metadata_def["meAccessibility"]["seDistribution"]["originalResource"] = output_filename
# adding type of layer
aggregationProcessing = "none"
if "3B42a_" in f:
aggregationProcessing = "anomaly"
elif "avg" in f:
aggregationProcessing = "avg"
metadata_def["meStatisticalProcessing"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"]["aggregationProcessing"] = aggregationProcessing;
# default style
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
if aggregationProcessing == "anomaly":
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = "rainfall_sadc_" + aggregationProcessing
else:
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = "rainfall_sadc"
# merging metadata to the base raster one
metadata_def = merge_layer_metadata("raster", metadata_def)
# "seCoverage" : {
# "coverageTime" : {
# "from" : 1328405808080,
# "to": 1328405808080
# },
# "coverageGeographic" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "world"}]
# },
# "coverageSector" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "MODISQ1"}]
# }
# }]
log.info(metadata_def)
# publish layer
print manager.publish_coverage(f, metadata_def)
def publish():
folders = glob.glob(input_folder)
for folder in folders:
files = glob.glob(folder + "/output/*.tif")
for f in files:
if "Average" in f:
print "Average"
log.info("--------------------------------")
log.info(f)
name = get_filename(f)
to_day = name[name.index("_") + 1:name.rindex("_")]
year = 1990
print d
print year
from_day = int(to_day) - 15
print from_day
from_date = calendar.timegm(
day_of_the_year_to_date(to_day, year).timetuple())
to_date = calendar.timegm(
day_of_the_year_to_date(from_day, year).timetuple())
#title = name.replace("", " ").upper()
title = "NDVI SADC Avg - " + to_day
else:
log.info("--------------------------------")
log.info(f)
name = get_filename(f)
d = name[name.index("_") + 1:name.rindex("_")]
year = d[0:4]
to_day = d[4:]
print d
print year
print to_day
from_day = int(to_day) - 15
print from_day
fromdate = day_of_the_year_to_date(to_day, year)
todate = day_of_the_year_to_date(from_day, year)
from_date = calendar.timegm(fromdate.timetuple())
to_date = calendar.timegm(todate.timetuple())
print from_date
print to_date
# get title name
#title = name.replace("_", " ").capitalize()
if "Anomaly" in f:
title = "NDVI SADC Anomaly - " + str(fromdate)
else:
title = "NDVI SADC - " + str(fromdate)
# get metadata
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
# Sample of Metadata json
log.info("Creating metadata")
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"][
"from"] = from_date
metadata_def["meContent"]["seCoverage"]["coverageTime"][
"to"] = to_date
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"][
"codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"][
"codes"] = [{
"code": "MODIS-NDVI-SADC"
}]
metadata_def["meContent"]["seCoverage"]["coverageSector"][
"codes"] = [{
"code": "MODIS-NDVI-SADC"
}]
# TODO: in theory should be the original file the onlineResource
metadata_def["meAccessibility"] = {}
metadata_def["meAccessibility"]["seDistribution"] = {}
metadata_def["meAccessibility"]["seDistribution"][
"onlineResource"] = f
# TODO: added new field for the original resource (should we have two different metadata?)
#metadata_def["meAccessibility"]["seDistribution"]["originalResource"] = output_filename
# adding type of layer
aggregationProcessing = "none"
if "Anomaly" in f:
aggregationProcessing = "anomaly"
elif "Average" in f:
aggregationProcessing = "avg"
metadata_def["meStatisticalProcessing"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"][
"seDataCompilation"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"][
"seDataCompilation"][
"aggregationProcessing"] = aggregationProcessing
# default style
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
if aggregationProcessing == "anomaly":
metadata_def["meSpatialRepresentation"]["seDefaultStyle"][
"name"] = "ndvi_" + aggregationProcessing
else:
metadata_def["meSpatialRepresentation"]["seDefaultStyle"][
"name"] = "ndvi"
# merging metadata to the base raster one
metadata_def = merge_layer_metadata("raster", metadata_def)
# "seCoverage" : {
# "coverageTime" : {
# "from" : 1328405808080,
# "to": 1328405808080
# },
# "coverageGeographic" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "world"}]
# },
# "coverageSector" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "MODISQ1"}]
# }
# }]
log.info(metadata_def)
# publish layer
print manager.publish_coverage(f, metadata_def)
def publish_layers():
files = glob.glob(output_folder + "output/*.tif")
for f in files:
log.info("--------------------------------")
name = get_filename(f)
log.info("name %s %s" % (name, f))
month = int(name[5:7])
log.info("month %s" % month)
year = None
try:
year = int(name[8:12])
except Exception, e:
log.error("year error")
# TODO: what year for the DA?
if year is None:
year = "2014"
# get title name
title = name.replace("_", " ").capitalize()
# get metadata
from_date, to_date = get_range_dates_metadata(month, year)
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
# Sample of Metadata json
log.info("Creating metadata")
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = title
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"][
"from"] = from_date
metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = to_date
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"][
"codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{
"code":
"TRMM"
}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{
"code":
"TRMM"
}]
# TODO: in theory should be the original file the onlineResource
metadata_def["meAccessibility"] = {}
metadata_def["meAccessibility"]["seDistribution"] = {}
metadata_def["meAccessibility"]["seDistribution"]["onlineResource"] = f
# TODO: added new field for the original resource (should we have two different metadata?)
#metadata_def["meAccessibility"]["seDistribution"]["originalResource"] = output_filename
# adding type of layer
aggregationProcessing = "none"
if "_avg" in name:
aggregationProcessing = "avg"
elif "_da" in name:
aggregationProcessing = "da"
metadata_def["meStatisticalProcessing"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"][
"seDataCompilation"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"][
"seDataCompilation"][
"aggregationProcessing"] = aggregationProcessing
# default style
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
if aggregationProcessing == "da":
metadata_def["meSpatialRepresentation"]["seDefaultStyle"][
"name"] = "rainfall_" + aggregationProcessing
else:
metadata_def["meSpatialRepresentation"]["seDefaultStyle"][
"name"] = "rainfall"
# merging metadata to the base raster one
metadata_def = merge_layer_metadata("raster", metadata_def)
# "seCoverage" : {
# "coverageTime" : {
# "from" : 1328405808080,
# "to": 1328405808080
# },
# "coverageGeographic" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "world"}]
# },
# "coverageSector" : {
# "codeList" : "...",
# "version" : "...",
# "codes" : [{"code" : "MODISQ1"}]
# }
# }]
log.info(metadata_def)
# publish layer
print manager.publish_coverage(f, metadata_def)
def publish():
path = "/home/vortex/Desktop/LAYERS/GHG/"
for dir in os.listdir(path):
filepath = os.path.join(path, dir, dir + ".geotiff")
p, fp, name = get_filename(filepath, True)
print p
print fp
print name
date = name[len(name)-4:]
product_code = name[:len(name)-5]
print date
print product_code
creationDate = calendar.timegm(datetime.datetime.now().timetuple())
from_date = datetime.datetime(int(date), int(1), 1)
to_date = datetime.datetime(int(date), int(12), 31)
# Sample of Metadata json
metadata_def = {}
metadata_def["title"] = {}
metadata_def["title"]["EN"] = name
metadata_def["creationDate"] = creationDate
metadata_def["meContent"] = {}
metadata_def["meContent"]["seCoverage"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"] = {}
metadata_def["meContent"]["seCoverage"]["coverageTime"]["from"] = from_date
metadata_def["meContent"]["seCoverage"]["coverageTime"]["to"] = to_date
metadata_def["meContent"]["seCoverage"]["coverageSector"] = {}
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codeList"] = "Products"
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : product_code}]
metadata_def["meContent"]["seCoverage"]["coverageSector"]["codes"] = [{"code" : product_code}]
# TODO: in theory should be the original file the onlineResource
metadata_def["meAccessibility"] = {}
metadata_def["meAccessibility"]["seDistribution"] = {}
# metadata_def["meAccessibility"]["seDistribution"]["onlineResource"] = "/media/vortex/16DE-3364/MODIS_250m.tif"
# TODO: added new field for the original resource (should we have two different metadata?)
#metadata_def["meAccessibility"]["seDistribution"]["originalResource"] = output_filename
# adding type of layer
aggregationProcessing = "none"
metadata_def["meStatisticalProcessing"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"] = {}
metadata_def["meStatisticalProcessing"]["seDatasource"]["seDataCompilation"]["aggregationProcessing"] = aggregationProcessing;
# default style
metadata_def["meSpatialRepresentation"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"] = {}
metadata_def["meSpatialRepresentation"]["seDefaultStyle"]["name"] = default_style
# merging metadata to the base raster one
metadata_def = merge_layer_metadata("raster", metadata_def)
#print manager.publish_coverage(filepath, metadata_def)
print manager.geoserver.set_default_style(name, default_style)
def _publish_shapefile(self, file_path, metadata_def=None, geoserver_def=None, overwrite=False, publish_on_geoserver=True, publish_metadata=True):
"""
@param file_path:
@param layer_def:
@param overwrite:
@return:
"""
try:
# layer_def = layer_def["featureType"]
# Unzip the shapefile
if file_path is not None:
shp_folder = filesystem.unzip(file_path)
shp_name = filesystem.get_filename(file_path)
# creating shp path
shp_folder_and_name = os.path.join(shp_folder, shp_name) + ".shp"
log.info(shp_folder_and_name)
# sanitize the layer_name
# name = sanitize_name(shp_name)
name = sanitize_name(metadata_def["title"]["EN"])
# getting the default workspace
workspace = self.geoserver.get_default_workspace()
if "workspace" in metadata_def["meSpatialRepresentation"]:
workspace = metadata_def["meSpatialRepresentation"]["workspace"]
# setting up the uid
if "uid" not in metadata_def:
metadata_def["uid"] = workspace + ":" + name
# publish shapefile on geoserver
# TODO: merge the metadata with the default vector metadata
if "name" not in geoserver_def:
geoserver_def["name"] = name
if "title" not in geoserver_def:
geoserver_def["title"] = name
if "workspace" not in geoserver_def:
geoserver_def["workspace"] = workspace
if "datastore" not in geoserver_def:
geoserver_def["datastore"] = self.geoserver.get_default_datastore()
# clean layer name
geoserver_def["name"] = sanitize_name(geoserver_def["name"])
# import the shapefile on postgis
# TODO: this should be another import check like: if publish_postgis_table is True
if publish_on_geoserver is True:
shapefile.import_shapefile(self.spatial_db, shp_folder_and_name, shp_name, True)
# publish on metadata
if publish_metadata is True:
self.metadata.db_metadata.insert_metadata(metadata_def)
# publish table on geoserver cluster
if publish_on_geoserver is True:
self.geoserver.publish_postgis_table(geoserver_def, True)
# remove files in input_shapefile
if file_path is not None:
filesystem.remove_folder(shp_folder)
except PGeoException, e:
log.error(e)
self.rollback_shapefile()
