def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts,
append_to_mosaic_name, mosaic_time_regex,
mosaic_time_value, time_presentation,
time_presentation_res,
time_presentation_default_value,
time_presentation_reference_value):
# The very first step is to rename the granule by adding the selected regex
# matching value to the filename.
f = spatial_files[0].base_file
dirname = os.path.dirname(f)
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
if not mosaic_time_regex:
mosaic_time_regex, mosaic_time_format = _get_time_regex(
spatial_files, basename)
# 0. A Time Regex is mandartory to validate the files
if not mosaic_time_regex:
raise UploadException(
_("Could not find any valid Time Regex for the Mosaic files."))
for spatial_file in spatial_files:
f = spatial_file.base_file
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
regexp = re.compile(mosaic_time_regex)
if regexp.match(head).groups():
mosaic_time_value = regexp.match(head).groups()[0]
head = head.replace(
regexp.match(head).groups()[0], '{mosaic_time_value}')
if mosaic_time_value:
dst_file = os.path.join(
dirname,
head.replace('{mosaic_time_value}', mosaic_time_value) + tail)
os.rename(f, dst_file)
spatial_file.base_file = dst_file
# We use the GeoServer REST APIs in order to create the ImageMosaic
# and later add the granule through the GeoServer Importer.
head = head.replace('{mosaic_time_value}', '')
head = re.sub('^[^a-zA-z]*|[^a-zA-Z]*$', '', head)
# 1. Create a zip file containing the ImageMosaic .properties files
# 1a. Let's check and prepare the DB based DataStore
cat = gs_catalog
workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE)
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
if not db_engine == 'postgis':
raise UploadException(_("Unsupported DataBase for Mosaics!"))
# dsname = ogc_server_settings.DATASTORE
dsname = db['NAME']
ds_exists = False
try:
ds = get_store(cat, dsname, workspace=workspace)
ds_exists = (ds is not None)
except FailedRequestError:
ds = cat.create_datastore(dsname, workspace=workspace)
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
ds.connection_parameters.update({
'validate connections':
'true',
'max connections':
'10',
'min connections':
'1',
'fetch size':
'1000',
'host':
db['HOST'],
'port':
db['PORT'] if isinstance(db['PORT'], basestring) else
str(db['PORT']) or '5432',
'database':
db['NAME'],
'user':
db['USER'],
'passwd':
db['PASSWORD'],
'dbtype':
db_engine
})
cat.save(ds)
ds = get_store(cat, dsname, workspace=workspace)
ds_exists = (ds is not None)
if not ds_exists:
raise UploadException(_("Unsupported DataBase for Mosaics!"))
context = {
"abs_path_flag":
"True",
"time_attr":
"time",
"aux_metadata_flag":
"False",
"mosaic_time_regex":
mosaic_time_regex,
"db_host":
db['HOST'],
"db_port":
db['PORT'],
"db_name":
db['NAME'],
"db_user":
db['USER'],
"db_password":
db['PASSWORD'],
"db_conn_timeout":
db['CONN_TOUT'] if 'CONN_TOUT' in db else "10",
"db_conn_min":
db['CONN_MIN'] if 'CONN_MIN' in db else "1",
"db_conn_max":
db['CONN_MAX'] if 'CONN_MAX' in db else "5",
"db_conn_validate":
db['CONN_VALIDATE'] if 'CONN_VALIDATE' in db else "true",
}
if mosaic_time_regex:
indexer_template = """AbsolutePath={abs_path_flag}
TimeAttribute={time_attr}
Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date
PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr})
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
timeregex_template = """regex=(?<=_)({mosaic_time_regex})"""
if not os.path.exists(dirname + '/timeregex.properties'):
with open(dirname + '/timeregex.properties',
'w') as timeregex_prop_file:
timeregex_prop_file.write(timeregex_template.format(**context))
else:
indexer_template = """AbsolutePath={abs_path_flag}
Schema= the_geom:Polygon,location:String,{time_attr}
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
datastore_template = r"""SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory
host={db_host}
port={db_port}
database={db_name}
user={db_user}
passwd={db_password}
Loose\ bbox=true
Estimated\ extends=false
validate\ connections={db_conn_validate}
Connection\ timeout={db_conn_timeout}
min\ connections={db_conn_min}
max\ connections={db_conn_max}"""
if not os.path.exists(dirname + '/indexer.properties'):
with open(dirname + '/indexer.properties', 'w') as indexer_prop_file:
indexer_prop_file.write(indexer_template.format(**context))
if not os.path.exists(dirname + '/datastore.properties'):
with open(dirname + '/datastore.properties',
'w') as datastore_prop_file:
datastore_prop_file.write(datastore_template.format(**context))
files_to_upload = []
if not append_to_mosaic_opts and spatial_files:
z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w")
for spatial_file in spatial_files:
f = spatial_file.base_file
dst_basename = os.path.basename(f)
dst_head, dst_tail = os.path.splitext(dst_basename)
if not files_to_upload:
# Let's import only the first granule
z.write(spatial_file.base_file, arcname=dst_head + dst_tail)
files_to_upload.append(spatial_file.base_file)
if os.path.exists(dirname + '/indexer.properties'):
z.write(dirname + '/indexer.properties',
arcname='indexer.properties')
if os.path.exists(dirname + '/datastore.properties'):
z.write(dirname + '/datastore.properties',
arcname='datastore.properties')
if mosaic_time_regex:
z.write(dirname + '/timeregex.properties',
arcname='timeregex.properties')
z.close()
# 2. Send a "create ImageMosaic" request to GeoServer through gs_config
cat._cache.clear()
# - name = name of the ImageMosaic (equal to the base_name)
# - data = abs path to the zip file
# - configure = parameter allows for future configuration after harvesting
name = head
data = open(dirname + '/' + head + '.zip', 'rb')
try:
cat.create_imagemosaic(name, data)
except ConflictingDataError:
# Trying to append granules to an existing mosaic
pass
# configure time as LIST
if mosaic_time_regex:
set_time_dimension(cat, name, workspace, time_presentation,
time_presentation_res,
time_presentation_default_value,
time_presentation_reference_value)
# - since GeoNode will upload the first granule again through the Importer, we need to /
# delete the one created by the gs_config
# mosaic_delete_first_granule(cat, name)
if len(spatial_files) > 1:
spatial_files = spatial_files[0]
return head, files_to_upload
else:
cat._cache.clear()
cat.reset()
# cat.reload()
return append_to_mosaic_name, files_to_upload
def import_imagemosaic_granules(
spatial_files,
append_to_mosaic_opts,
append_to_mosaic_name,
mosaic_time_regex,
mosaic_time_value,
time_presentation,
time_presentation_res,
time_presentation_default_value,
time_presentation_reference_value):
# The very first step is to rename the granule by adding the selected regex
# matching value to the filename.
f = spatial_files[0].base_file
dirname = os.path.dirname(f)
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
if not mosaic_time_regex:
mosaic_time_regex, mosaic_time_format = _get_time_regex(spatial_files, basename)
# 0. A Time Regex is mandartory to validate the files
if not mosaic_time_regex:
raise UploadException(_("Could not find any valid Time Regex for the Mosaic files."))
for spatial_file in spatial_files:
f = spatial_file.base_file
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
regexp = re.compile(mosaic_time_regex)
if regexp.match(head).groups():
mosaic_time_value = regexp.match(head).groups()[0]
head = head.replace(regexp.match(head).groups()[0], '{mosaic_time_value}')
if mosaic_time_value:
dst_file = os.path.join(
dirname,
head.replace('{mosaic_time_value}', mosaic_time_value) + tail)
os.rename(f, dst_file)
spatial_file.base_file = dst_file
# We use the GeoServer REST APIs in order to create the ImageMosaic
# and later add the granule through the GeoServer Importer.
head = head.replace('{mosaic_time_value}', '')
head = re.sub('^[^a-zA-z]*|[^a-zA-Z]*$', '', head)
# 1. Create a zip file containing the ImageMosaic .properties files
# 1a. Let's check and prepare the DB based DataStore
cat = gs_catalog
workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE)
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
if not db_engine == 'postgis':
raise UploadException(_("Unsupported DataBase for Mosaics!"))
# dsname = ogc_server_settings.DATASTORE
dsname = db['NAME']
ds_exists = False
try:
ds = get_store(cat, dsname, workspace=workspace)
ds_exists = (ds is not None)
except FailedRequestError:
ds = cat.create_datastore(dsname, workspace=workspace)
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
ds.connection_parameters.update(
{'validate connections': 'true',
'max connections': '10',
'min connections': '1',
'fetch size': '1000',
'host': db['HOST'],
'port': db['PORT'] if isinstance(
db['PORT'], basestring) else str(db['PORT']) or '5432',
'database': db['NAME'],
'user': db['USER'],
'passwd': db['PASSWORD'],
'dbtype': db_engine}
)
cat.save(ds)
ds = get_store(cat, dsname, workspace=workspace)
ds_exists = (ds is not None)
if not ds_exists:
raise UploadException(_("Unsupported DataBase for Mosaics!"))
context = {
"abs_path_flag": "True",
"time_attr": "time",
"aux_metadata_flag": "False",
"mosaic_time_regex": mosaic_time_regex,
"db_host": db['HOST'],
"db_port": db['PORT'],
"db_name": db['NAME'],
"db_user": db['USER'],
"db_password": db['PASSWORD'],
"db_conn_timeout": db['CONN_TOUT'] if 'CONN_TOUT' in db else "10",
"db_conn_min": db['CONN_MIN'] if 'CONN_MIN' in db else "1",
"db_conn_max": db['CONN_MAX'] if 'CONN_MAX' in db else "5",
"db_conn_validate": db['CONN_VALIDATE'] if 'CONN_VALIDATE' in db else "true",
}
if mosaic_time_regex:
indexer_template = """AbsolutePath={abs_path_flag}
TimeAttribute={time_attr}
Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date
PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr})
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
timeregex_template = """regex=(?<=_)({mosaic_time_regex})"""
if not os.path.exists(dirname + '/timeregex.properties'):
with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file:
timeregex_prop_file.write(timeregex_template.format(**context))
else:
indexer_template = """AbsolutePath={abs_path_flag}
Schema= the_geom:Polygon,location:String,{time_attr}
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory
host={db_host}
port={db_port}
database={db_name}
user={db_user}
passwd={db_password}
Loose\ bbox=true
Estimated\ extends=false
validate\ connections={db_conn_validate}
Connection\ timeout={db_conn_timeout}
min\ connections={db_conn_min}
max\ connections={db_conn_max}"""
if not os.path.exists(dirname + '/indexer.properties'):
with open(dirname + '/indexer.properties', 'w') as indexer_prop_file:
indexer_prop_file.write(indexer_template.format(**context))
if not os.path.exists(dirname + '/datastore.properties'):
with open(dirname + '/datastore.properties', 'w') as datastore_prop_file:
datastore_prop_file.write(datastore_template.format(**context))
files_to_upload = []
if not append_to_mosaic_opts and spatial_files:
z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w")
for spatial_file in spatial_files:
f = spatial_file.base_file
dst_basename = os.path.basename(f)
dst_head, dst_tail = os.path.splitext(dst_basename)
if not files_to_upload:
# Let's import only the first granule
z.write(spatial_file.base_file, arcname=dst_head + dst_tail)
files_to_upload.append(spatial_file.base_file)
if os.path.exists(dirname + '/indexer.properties'):
z.write(dirname + '/indexer.properties', arcname='indexer.properties')
if os.path.exists(dirname + '/datastore.properties'):
z.write(
dirname +
'/datastore.properties',
arcname='datastore.properties')
if mosaic_time_regex:
z.write(
dirname + '/timeregex.properties',
arcname='timeregex.properties')
z.close()
# 2. Send a "create ImageMosaic" request to GeoServer through gs_config
cat._cache.clear()
# - name = name of the ImageMosaic (equal to the base_name)
# - data = abs path to the zip file
# - configure = parameter allows for future configuration after harvesting
name = head
data = open(dirname + '/' + head + '.zip', 'rb')
try:
cat.create_imagemosaic(name, data)
except ConflictingDataError:
# Trying to append granules to an existing mosaic
pass
# configure time as LIST
if mosaic_time_regex:
set_time_dimension(
cat,
name,
workspace,
time_presentation,
time_presentation_res,
time_presentation_default_value,
time_presentation_reference_value)
# - since GeoNode will upload the first granule again through the Importer, we need to /
# delete the one created by the gs_config
# mosaic_delete_first_granule(cat, name)
if len(spatial_files) > 1:
spatial_files = spatial_files[0]
return head, files_to_upload
else:
cat._cache.clear()
cat.reset()
# cat.reload()
return append_to_mosaic_name, files_to_upload
def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name,
mosaic_time_regex, mosaic_time_value, time_presentation,
time_presentation_res, time_presentation_default_value,
time_presentation_reference_value):
# The very first step is to rename the granule by adding the selected regex
# matching value to the filename.
f = spatial_files[0].base_file
dirname = os.path.dirname(f)
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
dst_file = os.path.join(dirname, head.replace("_", "-") + "_" + mosaic_time_value + tail)
os.rename(f, dst_file)
spatial_files[0].base_file = dst_file
# We use the GeoServer REST APIs in order to create the ImageMosaic
# and later add the granule through the GeoServer Importer.
# 1. Create a zip file containing the ImageMosaic .properties files
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
if not db_engine == 'postgis':
raise UploadException("Unsupported DataBase for Mosaics!")
context = {
"abs_path_flag": "True",
"time_attr": "time",
"aux_metadata_flag": "False",
"mosaic_time_regex": mosaic_time_regex,
"db_host": db['HOST'],
"db_port": db['PORT'],
"db_name": db['NAME'],
"db_user": db['USER'],
"db_password": db['PASSWORD'],
"db_conn_timeout": db['CONN_TOUT'] or "10",
"db_conn_min": db['CONN_MIN'] or "1",
"db_conn_max": db['CONN_MAX'] or "5",
"db_conn_validate": db['CONN_VALIDATE'] or "true",
}
if mosaic_time_regex:
indexer_template = """AbsolutePath={abs_path_flag}
TimeAttribute={time_attr}
Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date
PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr})
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
timeregex_template = """regex=(?<=_)({mosaic_time_regex})"""
with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file:
timeregex_prop_file.write(timeregex_template.format(**context))
else:
indexer_template = """AbsolutePath={abs_path_flag}
Schema= the_geom:Polygon,location:String,{time_attr}
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory
host={db_host}
port={db_port}
database={db_name}
user={db_user}
passwd={db_password}
Loose\ bbox=true
Estimated\ extends=false
validate\ connections={db_conn_validate}
Connection\ timeout={db_conn_timeout}
min\ connections={db_conn_min}
max\ connections={db_conn_max}"""
with open(dirname + '/indexer.properties', 'w') as indexer_prop_file:
indexer_prop_file.write(indexer_template.format(**context))
with open(dirname + '/datastore.properties', 'w') as datastore_prop_file:
datastore_prop_file.write(datastore_template.format(**context))
if not append_to_mosaic_opts:
z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w")
z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail)
z.write(dirname + '/indexer.properties', arcname='indexer.properties')
z.write(dirname + '/datastore.properties', arcname='datastore.properties')
if mosaic_time_regex:
z.write(dirname + '/timeregex.properties', arcname='timeregex.properties')
z.close()
# 2. Send a "create ImageMosaic" request to GeoServer through gs_config
cat = gs_catalog
cat._cache.clear()
# - name = name of the ImageMosaic (equal to the base_name)
# - data = abs path to the zip file
# - configure = parameter allows for future configuration after harvesting
name = head
data = open(dirname + '/' + head + '.zip', 'rb')
cat.create_imagemosaic(name, data)
# configure time as LIST
if mosaic_time_regex:
set_time_dimension(cat, name, time_presentation, time_presentation_res,
time_presentation_default_value, time_presentation_reference_value)
# - since GeoNode will uploade the first granule again through the Importer, we need to /
# delete the one created by the gs_config
mosaic_delete_first_granule(cat, name)
return head
else:
cat = gs_catalog
cat._cache.clear()
cat.reset()
# cat.reload()
return append_to_mosaic_name
def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name,
mosaic_time_regex, mosaic_time_value, time_presentation,
time_presentation_res, time_presentation_default_value,
time_presentation_reference_value):
# The very first step is to rename the granule by adding the selected regex
# matching value to the filename.
f = spatial_files[0].base_file
dirname = os.path.dirname(f)
basename = os.path.basename(f)
head, tail = os.path.splitext(basename)
dst_file = os.path.join(dirname, head.replace("_", "-") + "_" + mosaic_time_value + tail)
os.rename(f, dst_file)
spatial_files[0].base_file = dst_file
# We use the GeoServer REST APIs in order to create the ImageMosaic
# and later add the granule through the GeoServer Importer.
# 1. Create a zip file containing the ImageMosaic .properties files
db = ogc_server_settings.datastore_db
db_engine = 'postgis' if \
'postgis' in db['ENGINE'] else db['ENGINE']
if not db_engine == 'postgis':
raise UploadException("Unsupported DataBase for Mosaics!")
context = {
"abs_path_flag": "True",
"time_attr": "time",
"aux_metadata_flag": "False",
"mosaic_time_regex": mosaic_time_regex,
"db_host": db['HOST'],
"db_port": db['PORT'],
"db_name": db['NAME'],
"db_user": db['USER'],
"db_password": db['PASSWORD'],
"db_conn_timeout": db['CONN_TOUT'] or "10",
"db_conn_min": db['CONN_MIN'] or "1",
"db_conn_max": db['CONN_MAX'] or "5",
"db_conn_validate": db['CONN_VALIDATE'] or "true",
}
if mosaic_time_regex:
indexer_template = """AbsolutePath={abs_path_flag}
TimeAttribute={time_attr}
Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date
PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr})
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
timeregex_template = """regex=(?<=_)({mosaic_time_regex})"""
with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file:
timeregex_prop_file.write(timeregex_template.format(**context))
else:
indexer_template = """AbsolutePath={abs_path_flag}
Schema= the_geom:Polygon,location:String,{time_attr}
CheckAuxiliaryMetadata={aux_metadata_flag}
SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi"""
datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory
host={db_host}
port={db_port}
database={db_name}
user={db_user}
passwd={db_password}
Loose\ bbox=true
Estimated\ extends=false
validate\ connections={db_conn_validate}
Connection\ timeout={db_conn_timeout}
min\ connections={db_conn_min}
max\ connections={db_conn_max}"""
with open(dirname + '/indexer.properties', 'w') as indexer_prop_file:
indexer_prop_file.write(indexer_template.format(**context))
with open(dirname + '/datastore.properties', 'w') as datastore_prop_file:
datastore_prop_file.write(datastore_template.format(**context))
if not append_to_mosaic_opts:
z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w")
z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail)
z.write(dirname + '/indexer.properties', arcname='indexer.properties')
z.write(dirname + '/datastore.properties', arcname='datastore.properties')
if mosaic_time_regex:
z.write(dirname + '/timeregex.properties', arcname='timeregex.properties')
z.close()
# 2. Send a "create ImageMosaic" request to GeoServer through gs_config
cat = gs_catalog
cat._cache.clear()
# - name = name of the ImageMosaic (equal to the base_name)
# - data = abs path to the zip file
# - configure = parameter allows for future configuration after harvesting
name = head
data = open(dirname + '/' + head + '.zip', 'rb')
cat.create_imagemosaic(name, data)
# configure time as LIST
if mosaic_time_regex:
set_time_dimension(cat, name, time_presentation, time_presentation_res,
time_presentation_default_value, time_presentation_reference_value)
# - since GeoNode will uploade the first granule again through the Importer, we need to /
# delete the one created by the gs_config
mosaic_delete_first_granule(cat, name)
return head
else:
cat = gs_catalog
cat._cache.clear()
cat.reset()
cat.reload()
return append_to_mosaic_name
