def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name, mosaic_time_regex, mosaic_time_value, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value): # The very first step is to rename the granule by adding the selected regex # matching value to the filename. f = spatial_files[0].base_file dirname = os.path.dirname(f) basename = os.path.basename(f) head, tail = os.path.splitext(basename) if not mosaic_time_regex: mosaic_time_regex, mosaic_time_format = _get_time_regex( spatial_files, basename) # 0. A Time Regex is mandartory to validate the files if not mosaic_time_regex: raise UploadException( _("Could not find any valid Time Regex for the Mosaic files.")) for spatial_file in spatial_files: f = spatial_file.base_file basename = os.path.basename(f) head, tail = os.path.splitext(basename) regexp = re.compile(mosaic_time_regex) if regexp.match(head).groups(): mosaic_time_value = regexp.match(head).groups()[0] head = head.replace( regexp.match(head).groups()[0], '{mosaic_time_value}') if mosaic_time_value: dst_file = os.path.join( dirname, head.replace('{mosaic_time_value}', mosaic_time_value) + tail) os.rename(f, dst_file) spatial_file.base_file = dst_file # We use the GeoServer REST APIs in order to create the ImageMosaic # and later add the granule through the GeoServer Importer. head = head.replace('{mosaic_time_value}', '') head = re.sub('^[^a-zA-z]*|[^a-zA-Z]*$', '', head) # 1. Create a zip file containing the ImageMosaic .properties files # 1a. Let's check and prepare the DB based DataStore cat = gs_catalog workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE) db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] if not db_engine == 'postgis': raise UploadException(_("Unsupported DataBase for Mosaics!")) # dsname = ogc_server_settings.DATASTORE dsname = db['NAME'] ds_exists = False try: ds = get_store(cat, dsname, workspace=workspace) ds_exists = (ds is not None) except FailedRequestError: ds = cat.create_datastore(dsname, workspace=workspace) db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] ds.connection_parameters.update({ 'validate connections': 'true', 'max connections': '10', 'min connections': '1', 'fetch size': '1000', 'host': db['HOST'], 'port': db['PORT'] if isinstance(db['PORT'], basestring) else str(db['PORT']) or '5432', 'database': db['NAME'], 'user': db['USER'], 'passwd': db['PASSWORD'], 'dbtype': db_engine }) cat.save(ds) ds = get_store(cat, dsname, workspace=workspace) ds_exists = (ds is not None) if not ds_exists: raise UploadException(_("Unsupported DataBase for Mosaics!")) context = { "abs_path_flag": "True", "time_attr": "time", "aux_metadata_flag": "False", "mosaic_time_regex": mosaic_time_regex, "db_host": db['HOST'], "db_port": db['PORT'], "db_name": db['NAME'], "db_user": db['USER'], "db_password": db['PASSWORD'], "db_conn_timeout": db['CONN_TOUT'] if 'CONN_TOUT' in db else "10", "db_conn_min": db['CONN_MIN'] if 'CONN_MIN' in db else "1", "db_conn_max": db['CONN_MAX'] if 'CONN_MAX' in db else "5", "db_conn_validate": db['CONN_VALIDATE'] if 'CONN_VALIDATE' in db else "true", } if mosaic_time_regex: indexer_template = """AbsolutePath={abs_path_flag} TimeAttribute={time_attr} Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" if not os.path.exists(dirname + '/timeregex.properties'): with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: timeregex_prop_file.write(timeregex_template.format(**context)) else: indexer_template = """AbsolutePath={abs_path_flag} Schema= the_geom:Polygon,location:String,{time_attr} CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" datastore_template = r"""SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory host={db_host} port={db_port} database={db_name} user={db_user} passwd={db_password} Loose\ bbox=true Estimated\ extends=false validate\ connections={db_conn_validate} Connection\ timeout={db_conn_timeout} min\ connections={db_conn_min} max\ connections={db_conn_max}""" if not os.path.exists(dirname + '/indexer.properties'): with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: indexer_prop_file.write(indexer_template.format(**context)) if not os.path.exists(dirname + '/datastore.properties'): with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: datastore_prop_file.write(datastore_template.format(**context)) files_to_upload = [] if not append_to_mosaic_opts and spatial_files: z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") for spatial_file in spatial_files: f = spatial_file.base_file dst_basename = os.path.basename(f) dst_head, dst_tail = os.path.splitext(dst_basename) if not files_to_upload: # Let's import only the first granule z.write(spatial_file.base_file, arcname=dst_head + dst_tail) files_to_upload.append(spatial_file.base_file) if os.path.exists(dirname + '/indexer.properties'): z.write(dirname + '/indexer.properties', arcname='indexer.properties') if os.path.exists(dirname + '/datastore.properties'): z.write(dirname + '/datastore.properties', arcname='datastore.properties') if mosaic_time_regex: z.write(dirname + '/timeregex.properties', arcname='timeregex.properties') z.close() # 2. Send a "create ImageMosaic" request to GeoServer through gs_config cat._cache.clear() # - name = name of the ImageMosaic (equal to the base_name) # - data = abs path to the zip file # - configure = parameter allows for future configuration after harvesting name = head data = open(dirname + '/' + head + '.zip', 'rb') try: cat.create_imagemosaic(name, data) except ConflictingDataError: # Trying to append granules to an existing mosaic pass # configure time as LIST if mosaic_time_regex: set_time_dimension(cat, name, workspace, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value) # - since GeoNode will upload the first granule again through the Importer, we need to / # delete the one created by the gs_config # mosaic_delete_first_granule(cat, name) if len(spatial_files) > 1: spatial_files = spatial_files[0] return head, files_to_upload else: cat._cache.clear() cat.reset() # cat.reload() return append_to_mosaic_name, files_to_upload
def import_imagemosaic_granules( spatial_files, append_to_mosaic_opts, append_to_mosaic_name, mosaic_time_regex, mosaic_time_value, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value): # The very first step is to rename the granule by adding the selected regex # matching value to the filename. f = spatial_files[0].base_file dirname = os.path.dirname(f) basename = os.path.basename(f) head, tail = os.path.splitext(basename) if not mosaic_time_regex: mosaic_time_regex, mosaic_time_format = _get_time_regex(spatial_files, basename) # 0. A Time Regex is mandartory to validate the files if not mosaic_time_regex: raise UploadException(_("Could not find any valid Time Regex for the Mosaic files.")) for spatial_file in spatial_files: f = spatial_file.base_file basename = os.path.basename(f) head, tail = os.path.splitext(basename) regexp = re.compile(mosaic_time_regex) if regexp.match(head).groups(): mosaic_time_value = regexp.match(head).groups()[0] head = head.replace(regexp.match(head).groups()[0], '{mosaic_time_value}') if mosaic_time_value: dst_file = os.path.join( dirname, head.replace('{mosaic_time_value}', mosaic_time_value) + tail) os.rename(f, dst_file) spatial_file.base_file = dst_file # We use the GeoServer REST APIs in order to create the ImageMosaic # and later add the granule through the GeoServer Importer. head = head.replace('{mosaic_time_value}', '') head = re.sub('^[^a-zA-z]*|[^a-zA-Z]*$', '', head) # 1. Create a zip file containing the ImageMosaic .properties files # 1a. Let's check and prepare the DB based DataStore cat = gs_catalog workspace = cat.get_workspace(settings.DEFAULT_WORKSPACE) db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] if not db_engine == 'postgis': raise UploadException(_("Unsupported DataBase for Mosaics!")) # dsname = ogc_server_settings.DATASTORE dsname = db['NAME'] ds_exists = False try: ds = get_store(cat, dsname, workspace=workspace) ds_exists = (ds is not None) except FailedRequestError: ds = cat.create_datastore(dsname, workspace=workspace) db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] ds.connection_parameters.update( {'validate connections': 'true', 'max connections': '10', 'min connections': '1', 'fetch size': '1000', 'host': db['HOST'], 'port': db['PORT'] if isinstance( db['PORT'], basestring) else str(db['PORT']) or '5432', 'database': db['NAME'], 'user': db['USER'], 'passwd': db['PASSWORD'], 'dbtype': db_engine} ) cat.save(ds) ds = get_store(cat, dsname, workspace=workspace) ds_exists = (ds is not None) if not ds_exists: raise UploadException(_("Unsupported DataBase for Mosaics!")) context = { "abs_path_flag": "True", "time_attr": "time", "aux_metadata_flag": "False", "mosaic_time_regex": mosaic_time_regex, "db_host": db['HOST'], "db_port": db['PORT'], "db_name": db['NAME'], "db_user": db['USER'], "db_password": db['PASSWORD'], "db_conn_timeout": db['CONN_TOUT'] if 'CONN_TOUT' in db else "10", "db_conn_min": db['CONN_MIN'] if 'CONN_MIN' in db else "1", "db_conn_max": db['CONN_MAX'] if 'CONN_MAX' in db else "5", "db_conn_validate": db['CONN_VALIDATE'] if 'CONN_VALIDATE' in db else "true", } if mosaic_time_regex: indexer_template = """AbsolutePath={abs_path_flag} TimeAttribute={time_attr} Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" if not os.path.exists(dirname + '/timeregex.properties'): with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: timeregex_prop_file.write(timeregex_template.format(**context)) else: indexer_template = """AbsolutePath={abs_path_flag} Schema= the_geom:Polygon,location:String,{time_attr} CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory host={db_host} port={db_port} database={db_name} user={db_user} passwd={db_password} Loose\ bbox=true Estimated\ extends=false validate\ connections={db_conn_validate} Connection\ timeout={db_conn_timeout} min\ connections={db_conn_min} max\ connections={db_conn_max}""" if not os.path.exists(dirname + '/indexer.properties'): with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: indexer_prop_file.write(indexer_template.format(**context)) if not os.path.exists(dirname + '/datastore.properties'): with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: datastore_prop_file.write(datastore_template.format(**context)) files_to_upload = [] if not append_to_mosaic_opts and spatial_files: z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") for spatial_file in spatial_files: f = spatial_file.base_file dst_basename = os.path.basename(f) dst_head, dst_tail = os.path.splitext(dst_basename) if not files_to_upload: # Let's import only the first granule z.write(spatial_file.base_file, arcname=dst_head + dst_tail) files_to_upload.append(spatial_file.base_file) if os.path.exists(dirname + '/indexer.properties'): z.write(dirname + '/indexer.properties', arcname='indexer.properties') if os.path.exists(dirname + '/datastore.properties'): z.write( dirname + '/datastore.properties', arcname='datastore.properties') if mosaic_time_regex: z.write( dirname + '/timeregex.properties', arcname='timeregex.properties') z.close() # 2. Send a "create ImageMosaic" request to GeoServer through gs_config cat._cache.clear() # - name = name of the ImageMosaic (equal to the base_name) # - data = abs path to the zip file # - configure = parameter allows for future configuration after harvesting name = head data = open(dirname + '/' + head + '.zip', 'rb') try: cat.create_imagemosaic(name, data) except ConflictingDataError: # Trying to append granules to an existing mosaic pass # configure time as LIST if mosaic_time_regex: set_time_dimension( cat, name, workspace, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value) # - since GeoNode will upload the first granule again through the Importer, we need to / # delete the one created by the gs_config # mosaic_delete_first_granule(cat, name) if len(spatial_files) > 1: spatial_files = spatial_files[0] return head, files_to_upload else: cat._cache.clear() cat.reset() # cat.reload() return append_to_mosaic_name, files_to_upload
def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name, mosaic_time_regex, mosaic_time_value, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value): # The very first step is to rename the granule by adding the selected regex # matching value to the filename. f = spatial_files[0].base_file dirname = os.path.dirname(f) basename = os.path.basename(f) head, tail = os.path.splitext(basename) dst_file = os.path.join(dirname, head.replace("_", "-") + "_" + mosaic_time_value + tail) os.rename(f, dst_file) spatial_files[0].base_file = dst_file # We use the GeoServer REST APIs in order to create the ImageMosaic # and later add the granule through the GeoServer Importer. # 1. Create a zip file containing the ImageMosaic .properties files db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] if not db_engine == 'postgis': raise UploadException("Unsupported DataBase for Mosaics!") context = { "abs_path_flag": "True", "time_attr": "time", "aux_metadata_flag": "False", "mosaic_time_regex": mosaic_time_regex, "db_host": db['HOST'], "db_port": db['PORT'], "db_name": db['NAME'], "db_user": db['USER'], "db_password": db['PASSWORD'], "db_conn_timeout": db['CONN_TOUT'] or "10", "db_conn_min": db['CONN_MIN'] or "1", "db_conn_max": db['CONN_MAX'] or "5", "db_conn_validate": db['CONN_VALIDATE'] or "true", } if mosaic_time_regex: indexer_template = """AbsolutePath={abs_path_flag} TimeAttribute={time_attr} Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: timeregex_prop_file.write(timeregex_template.format(**context)) else: indexer_template = """AbsolutePath={abs_path_flag} Schema= the_geom:Polygon,location:String,{time_attr} CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory host={db_host} port={db_port} database={db_name} user={db_user} passwd={db_password} Loose\ bbox=true Estimated\ extends=false validate\ connections={db_conn_validate} Connection\ timeout={db_conn_timeout} min\ connections={db_conn_min} max\ connections={db_conn_max}""" with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: indexer_prop_file.write(indexer_template.format(**context)) with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: datastore_prop_file.write(datastore_template.format(**context)) if not append_to_mosaic_opts: z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail) z.write(dirname + '/indexer.properties', arcname='indexer.properties') z.write(dirname + '/datastore.properties', arcname='datastore.properties') if mosaic_time_regex: z.write(dirname + '/timeregex.properties', arcname='timeregex.properties') z.close() # 2. Send a "create ImageMosaic" request to GeoServer through gs_config cat = gs_catalog cat._cache.clear() # - name = name of the ImageMosaic (equal to the base_name) # - data = abs path to the zip file # - configure = parameter allows for future configuration after harvesting name = head data = open(dirname + '/' + head + '.zip', 'rb') cat.create_imagemosaic(name, data) # configure time as LIST if mosaic_time_regex: set_time_dimension(cat, name, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value) # - since GeoNode will uploade the first granule again through the Importer, we need to / # delete the one created by the gs_config mosaic_delete_first_granule(cat, name) return head else: cat = gs_catalog cat._cache.clear() cat.reset() # cat.reload() return append_to_mosaic_name
def import_imagemosaic_granules(spatial_files, append_to_mosaic_opts, append_to_mosaic_name, mosaic_time_regex, mosaic_time_value, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value): # The very first step is to rename the granule by adding the selected regex # matching value to the filename. f = spatial_files[0].base_file dirname = os.path.dirname(f) basename = os.path.basename(f) head, tail = os.path.splitext(basename) dst_file = os.path.join(dirname, head.replace("_", "-") + "_" + mosaic_time_value + tail) os.rename(f, dst_file) spatial_files[0].base_file = dst_file # We use the GeoServer REST APIs in order to create the ImageMosaic # and later add the granule through the GeoServer Importer. # 1. Create a zip file containing the ImageMosaic .properties files db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ 'postgis' in db['ENGINE'] else db['ENGINE'] if not db_engine == 'postgis': raise UploadException("Unsupported DataBase for Mosaics!") context = { "abs_path_flag": "True", "time_attr": "time", "aux_metadata_flag": "False", "mosaic_time_regex": mosaic_time_regex, "db_host": db['HOST'], "db_port": db['PORT'], "db_name": db['NAME'], "db_user": db['USER'], "db_password": db['PASSWORD'], "db_conn_timeout": db['CONN_TOUT'] or "10", "db_conn_min": db['CONN_MIN'] or "1", "db_conn_max": db['CONN_MAX'] or "5", "db_conn_validate": db['CONN_VALIDATE'] or "true", } if mosaic_time_regex: indexer_template = """AbsolutePath={abs_path_flag} TimeAttribute={time_attr} Schema= the_geom:Polygon,location:String,{time_attr}:java.util.Date PropertyCollectors=TimestampFileNameExtractorSPI[timeregex]({time_attr}) CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" timeregex_template = """regex=(?<=_)({mosaic_time_regex})""" with open(dirname + '/timeregex.properties', 'w') as timeregex_prop_file: timeregex_prop_file.write(timeregex_template.format(**context)) else: indexer_template = """AbsolutePath={abs_path_flag} Schema= the_geom:Polygon,location:String,{time_attr} CheckAuxiliaryMetadata={aux_metadata_flag} SuggestedSPI=it.geosolutions.imageioimpl.plugins.tiff.TIFFImageReaderSpi""" datastore_template = """SPI=org.geotools.data.postgis.PostgisNGDataStoreFactory host={db_host} port={db_port} database={db_name} user={db_user} passwd={db_password} Loose\ bbox=true Estimated\ extends=false validate\ connections={db_conn_validate} Connection\ timeout={db_conn_timeout} min\ connections={db_conn_min} max\ connections={db_conn_max}""" with open(dirname + '/indexer.properties', 'w') as indexer_prop_file: indexer_prop_file.write(indexer_template.format(**context)) with open(dirname + '/datastore.properties', 'w') as datastore_prop_file: datastore_prop_file.write(datastore_template.format(**context)) if not append_to_mosaic_opts: z = zipfile.ZipFile(dirname + '/' + head + '.zip', "w") z.write(dst_file, arcname=head + "_" + mosaic_time_value + tail) z.write(dirname + '/indexer.properties', arcname='indexer.properties') z.write(dirname + '/datastore.properties', arcname='datastore.properties') if mosaic_time_regex: z.write(dirname + '/timeregex.properties', arcname='timeregex.properties') z.close() # 2. Send a "create ImageMosaic" request to GeoServer through gs_config cat = gs_catalog cat._cache.clear() # - name = name of the ImageMosaic (equal to the base_name) # - data = abs path to the zip file # - configure = parameter allows for future configuration after harvesting name = head data = open(dirname + '/' + head + '.zip', 'rb') cat.create_imagemosaic(name, data) # configure time as LIST if mosaic_time_regex: set_time_dimension(cat, name, time_presentation, time_presentation_res, time_presentation_default_value, time_presentation_reference_value) # - since GeoNode will uploade the first granule again through the Importer, we need to / # delete the one created by the gs_config mosaic_delete_first_granule(cat, name) return head else: cat = gs_catalog cat._cache.clear() cat.reset() cat.reload() return append_to_mosaic_name