class DataProviderTaskRecord(UIDMixin, TimeStampedModelMixin,
TimeTrackingModelMixin):
"""
The DataProviderTaskRecord stores the task information for a specific provider.
"""
from eventkit_cloud.jobs.models import MapImageSnapshot
name = models.CharField(max_length=100, blank=True)
slug = LowerCaseCharField(max_length=40, default="")
provider = models.ForeignKey(DataProvider,
on_delete=models.CASCADE,
related_name="task_record_providers",
null=True,
blank=True)
run = models.ForeignKey(ExportRun,
related_name="data_provider_task_records",
on_delete=models.CASCADE)
status = models.CharField(blank=True, max_length=20, db_index=True)
display = models.BooleanField(default=False)
estimated_size = models.FloatField(null=True, blank=True)
estimated_duration = models.FloatField(null=True, blank=True)
preview = models.ForeignKey(MapImageSnapshot,
blank=True,
null=True,
on_delete=models.SET_NULL,
help_text="A preview for a provider task.")
class Meta:
ordering = ["name"]
managed = True
db_table = "data_provider_task_records"
unique_together = ["provider", "run"]
def __str__(self):
return "DataProviderTaskRecord uid: {0}".format(str(self.uid))
def clone(self, run: ExportRun):
"""
The ExportRun needs to be a **new** run, otherwise integrity errors will happen.
"""
export_task_records = list(self.tasks.all())
preview = self.preview
self.id = None
self.uid = None
self.run = run
self.save()
for export_task_record in export_task_records:
etr = export_task_record.clone(self)
if not self.tasks.filter(id=etr.id).exists():
self.tasks.add(etr)
if preview:
self.preview = preview.clone()
return self
class License(TimeStampedModelMixin):
"""
Model to hold license information to be used with DataProviders.
"""
slug = LowerCaseCharField(max_length=40, unique=True, default="")
name = models.CharField(max_length=100, db_index=True)
text = models.TextField(default="")
def __str__(self):
return "{0}".format(self.name)
class ExportFormat(UIDMixin, TimeStampedModelMixin):
"""
Model for a ExportFormat.
"""
safe_kwargs = [
"name",
"slug",
"description",
"cmd",
]
name = models.CharField(max_length=100)
slug = LowerCaseCharField(max_length=20, unique=True, default="")
description = models.CharField(max_length=255)
options = models.JSONField(default=dict, null=True, blank=True)
objects = models.Manager()
supported_projections = models.ManyToManyField(
Projection, related_name="supported_projections")
class Meta: # pragma: no cover
managed = True
db_table = "export_formats"
def __str__(self):
return "{0}".format(self.name)
@classmethod
def get_or_create(cls, **kwargs):
blacklisted_keys = []
created = False
for _key in kwargs:
if _key not in cls.safe_kwargs:
blacklisted_keys.append(_key)
for _key in blacklisted_keys:
del kwargs[_key]
try:
format = cls.objects.get(slug=kwargs.get("slug").lower())
except ObjectDoesNotExist:
format = cls.objects.create(**kwargs)
created = True
return format, created
def get_supported_projection_list(self) -> List[int]:
supported_projections = self.supported_projections.all().values_list(
"srid", flat=True)
return list(supported_projections)
class Topic(UIDMixin, TimeStampedModelMixin): # type: ignore
"""
Model for a Topic
"""
name = models.CharField(verbose_name="Topic Name",
unique=True,
max_length=100)
slug = LowerCaseCharField(max_length=40, unique=True, default="")
providers = models.ManyToManyField(DataProvider, related_name="topics")
topic_description = models.TextField(
verbose_name="Description",
null=True,
default="",
blank=True,
help_text=
"This information is used to provide information about the Topic.",
)
class Meta:
verbose_name_plural = "Topics"
def __str__(self):
return "{0}".format(self.name)
class DataProvider(UIDMixin, TimeStampedModelMixin, CachedModelMixin):
"""
Model for a DataProvider.
"""
name = models.CharField(verbose_name="Service Name",
unique=True,
max_length=100)
slug = LowerCaseCharField(max_length=40, unique=True, default="")
label = models.CharField(verbose_name="Label",
max_length=100,
null=True,
blank=True)
url = models.CharField(
verbose_name="Service URL",
max_length=1000,
null=True,
default="",
blank=True,
help_text=
"The SERVICE_URL is used as the endpoint for WFS, OSM, and WCS services. It is "
"also used to check availability for all OGC services. If you are adding a TMS "
"service, please provide a link to a single tile, but with the coordinate numbers "
"replaced by {z}, {y}, and {x}. Example: https://tiles.your-geospatial-site.com/"
"tiles/default/{z}/{y}/{x}.png",
)
preview_url = models.CharField(
verbose_name="Preview URL",
max_length=1000,
null=True,
default="",
blank=True,
help_text=
"This url will be served to the front end for displaying in the map.",
)
service_copyright = models.CharField(
verbose_name="Copyright",
max_length=2000,
null=True,
default="",
blank=True,
help_text=
"This information is used to display relevant copyright information.",
)
service_description = models.TextField(
verbose_name="Description",
null=True,
default="",
blank=True,
help_text=
"This information is used to provide information about the service.",
)
layer = models.CharField(verbose_name="Service Layer",
max_length=100,
null=True,
blank=True)
export_provider_type = models.ForeignKey(DataProviderType,
verbose_name="Service Type",
null=True,
on_delete=models.CASCADE)
max_selection = models.DecimalField(
verbose_name="Max selection area",
default=250,
max_digits=12,
decimal_places=3,
help_text=
"This is the maximum area in square kilometers that can be exported "
"from this provider in a single DataPack.",
)
level_from = models.IntegerField(
verbose_name="Seed from level",
default=0,
null=True,
blank=True,
help_text=
"This determines the starting zoom level the tile export will seed from.",
)
level_to = models.IntegerField(
verbose_name="Seed to level",
default=10,
null=True,
blank=True,
help_text=
"This determines the highest zoom level the tile export will seed to.",
)
config = models.TextField(
default="",
null=True,
blank=True,
verbose_name="Configuration",
help_text=
"""WMS, TMS, WMTS, and ArcGIS-Raster require a MapProxy YAML configuration
with a Sources key of imagery and a Service Layer name of imagery; the validator also
requires a layers section, but this isn't used.
OSM Services also require a YAML configuration.""",
)
DATA_TYPES = [
(GeospatialDataType.VECTOR.value, ("Vector")),
(GeospatialDataType.RASTER.value, ("Raster")),
(GeospatialDataType.ELEVATION.value, ("Elevation")),
(GeospatialDataType.MESH.value, ("Mesh")),
(GeospatialDataType.POINT_CLOUD.value, ("Point Cloud")),
]
data_type = models.CharField(
choices=DATA_TYPES,
max_length=20,
verbose_name="Data Type",
null=True,
default="",
blank=True,
help_text="The type of data provided (e.g. elevation, raster, vector)",
)
user = models.ForeignKey(User,
related_name="+",
null=True,
default=None,
blank=True,
on_delete=models.CASCADE)
license = models.ForeignKey(License,
related_name="data_providers",
null=True,
blank=True,
default=None,
on_delete=models.CASCADE)
zip = models.BooleanField(default=False)
display = models.BooleanField(default=False)
thumbnail = models.ForeignKey(
MapImageSnapshot,
blank=True,
null=True,
on_delete=models.SET_NULL,
help_text="A thumbnail image generated to give a high level"
" preview of what a provider's data looks like.",
)
attribute_class = models.ForeignKey(
AttributeClass,
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name="data_providers",
help_text=
"The attribute class is used to limit users access to resources using this data provider.",
)
the_geom = models.MultiPolygonField(
verbose_name="Covered Area",
srid=4326,
default=
"SRID=4326;MultiPolygon (((-180 -90,180 -90,180 90,-180 90,-180 -90)))",
)
class Meta: # pragma: no cover
managed = True
db_table = "export_provider"
# Check if config changed to updated geometry
__config: str = None
__url: str = None
__layer: str = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__config = self.config
self.__url = self.url
self.__layer = self.layer
def update_geom(self):
from eventkit_cloud.tasks.helpers import download_data
from eventkit_cloud.ui.helpers import file_to_geojson
geometry = None
if self.config != self.__config:
orig_extent_url = load_provider_config(
self.__config).get("extent_url")
config = load_provider_config(self.config)
extent_url = config.get("extent_url")
if extent_url and extent_url != orig_extent_url:
random_uuid = uuid.uuid4()
session = get_or_update_session(**config)
if not extent_url:
return
output_file = download_data(task_uid=str(random_uuid),
input_url=extent_url,
session=session)
geojson = file_to_geojson(output_file)
geojson_geometry = geojson.get("geometry") or geojson.get(
"features", [{}])[0].get("geometry")
geometry = GEOSGeometry(json.dumps(geojson_geometry),
srid=4326)
elif (self.url != self.__url) or (self.layer != self.__layer):
try:
client = self.get_service_client()
geometry = client.download_geometry()
except AttributeError as e:
# TODO: This fails in tests. How to handle failure to update geometry?
logger.info(e, exc_info=True)
if geometry:
self.the_geom = convert_polygon(geometry)
def get_service_client(self) -> GisClient:
url = self.url
if not self.url and "osm" in self.export_provider_type.type_name:
logger.error(
"Use of settings.OVERPASS_API_URL is deprecated and will be removed in 1.13"
)
url = settings.OVERPASS_API_URL
Client = get_client(self.export_provider_type.type_name)
config = None
if self.config:
config = load_provider_config(self.config)
return Client(url,
self.layer,
aoi_geojson=None,
slug=self.slug,
config=config)
def check_status(self, aoi_geojson: dict = None):
try:
client = self.get_service_client()
response = client.check(aoi_geojson=aoi_geojson)
except Exception as e:
logger.error(e, exc_info=True)
response = get_status_result(CheckResult.UNKNOWN_ERROR)
logger.error(
f"An exception occurred while checking the {self.name} provider.",
exc_info=True)
logger.info(f"Status of provider '{self.name}': {response}")
return response
def save(self, force_insert=False, force_update=False, *args, **kwargs):
# Something is closing the database connection which is raising an error.
# Using a separate process allows the connection to be closed in separate process while leaving it open.
proc = multiprocessing.dummy.Process(target=self.update_geom)
proc.start()
proc.join()
if not self.slug:
self.slug = self.name.replace(" ", "_").lower()
if len(self.slug) > 40:
self.slug = self.slug[0:39]
cache.delete(f"base-config-{self.slug}")
self.update_export_formats()
super(DataProvider, self).save(force_insert, force_update, *args,
**kwargs)
def update_export_formats(self):
# TODO: Refactor utils/ogc_apiprocess into services.
from eventkit_cloud.utils.ogcapi_process import get_process_formats
process_formats = get_process_formats(self)
logger.info(f"Process_formats: {process_formats}")
for process_format in process_formats:
export_format, created = ExportFormat.get_or_create(
**process_format)
if created:
# Use the value from process format which might be case sensitive,
# TODO: will likley run into issues if two remote services use same spelling and are case sensitive.
export_format.options = {
"value": process_format.get("slug"),
"providers": [self.slug],
"proxy": True
}
export_format.supported_projections.add(
Projection.objects.get(srid=4326))
else:
providers = export_format.options.get("providers")
if providers:
providers = list(set(providers + [self.slug]))
export_format.options["providers"] = providers
else:
export_format.options = {
"value": export_format.slug,
"providers": [self.slug],
"proxy": True
}
export_format.save()
def __str__(self):
return "{0}".format(self.name)
@property
def metadata(self):
from eventkit_cloud.utils.mapproxy import get_mapproxy_metadata_url
if not self.config:
return None
config = yaml.load(self.config, Loader=CLoader)
url = config.get("sources", {}).get("info", {}).get("req",
{}).get("url")
type = config.get("sources", {}).get("info", {}).get("type")
if url:
return {"url": get_mapproxy_metadata_url(self.slug), "type": type}
@property
def footprint_url(self):
from eventkit_cloud.utils.mapproxy import get_mapproxy_footprint_url
if not self.config:
return None
config = yaml.load(self.config, Loader=CLoader)
url = config.get("sources", {}).get("footprint", {}).get("req",
{}).get("url")
if url:
return get_mapproxy_footprint_url(self.slug)
@property
def layers(self) -> LayersDescription:
"""
Used to populate the list of vector layers, typically for contextual or styling information.
:return: A list of layer names.
"""
if self.data_type != GeospatialDataType.VECTOR.value:
return {}
if self.config:
config = clean_config(str(self.config))
# As of EK 1.9.0 only vectors support multiple layers in a single provider
if self.export_provider_type.type_name in ["osm", "osm-generic"]:
return config
elif config.get("vector_layers"):
return {
layer.get("name"): layer
for layer in config.get("vector_layers", {})
}
# Often layer names are configured using an index number but this number is not very
# useful when using the data so fall back to the slug which should be more meaningful.
if not self.layer: # check for NoneType or empty string
# TODO: support other service types
if self.export_provider_type.type_name in ["arcgis-feature"]:
return self.get_service_client().get_layers()
else:
return {self.slug: {"url": self.url, "name": self.slug}}
try:
int(self.layer) # noqa
return {
self.slug: {
"url": self.url,
"name": self.slug
}
} # self.layer is an integer, so use the slug for better context.
except ValueError:
return {
self.layer: {
"url": self.url,
"name": self.layer
}
} # If we got here, layer is not None or an integer so use that.
def get_use_bbox(self):
if self.export_provider_type is not None:
return self.export_provider_type.use_bbox
else:
return False
"""
Max datasize is the size in megabytes.
"""
@property
def max_data_size(self):
config = yaml.load(self.config, Loader=CLoader)
return None if config is None else config.get("max_data_size", None)
def get_max_data_size(self, user=None):
if not user:
return self.max_data_size
# the usersizerule set is looped instead of using a queryset filter so that it can be prefetched.
if user:
user_size_rule = list(
filter(lambda user_size_rule: user_size_rule.user == user,
self.usersizerule_set.all()))
if user_size_rule:
return user_size_rule[0].max_data_size
return self.max_data_size
def get_max_selection_size(self, user=None):
if not user:
return self.max_selection
# the usersizerule set is looped instead of using a queryset filter so that it can be prefetched.
if user:
user_size_rule = list(
filter(lambda user_size_rule: user_size_rule.user == user,
self.usersizerule_set.all()))
if user_size_rule:
return user_size_rule[0].max_selection_size
return self.max_selection
def get_data_type(self) -> str:
"""
This is used to populate the run metadata with special types for OSM and NOME.
This is used for custom cartography,
and should be removed if custom cartography is made configurable.
:param data_provider:
:return:
"""
if self.slug.lower() in ["nome", "osm"]:
return self.slug.lower()
else:
return str(self.data_type)
class DataProvider(UIDMixin, TimeStampedModelMixin, CachedModelMixin):
"""
Model for a DataProvider.
"""
name = models.CharField(verbose_name="Service Name", unique=True, max_length=100)
slug = LowerCaseCharField(max_length=40, unique=True, default="")
label = models.CharField(verbose_name="Label", max_length=100, null=True, blank=True)
url = models.CharField(
verbose_name="Service URL",
max_length=1000,
null=True,
default="",
blank=True,
help_text="The SERVICE_URL is used as the endpoint for WFS, OSM, and WCS services. It is "
"also used to check availability for all OGC services. If you are adding a TMS "
"service, please provide a link to a single tile, but with the coordinate numbers "
"replaced by {z}, {y}, and {x}. Example: https://tiles.your-geospatial-site.com/"
"tiles/default/{z}/{y}/{x}.png",
)
preview_url = models.CharField(
verbose_name="Preview URL",
max_length=1000,
null=True,
default="",
blank=True,
help_text="This url will be served to the front end for displaying in the map.",
)
service_copyright = models.CharField(
verbose_name="Copyright",
max_length=2000,
null=True,
default="",
blank=True,
help_text="This information is used to display relevant copyright information.",
)
service_description = models.TextField(
verbose_name="Description",
null=True,
default="",
blank=True,
help_text="This information is used to provide information about the service.",
)
layer = models.CharField(verbose_name="Service Layer", max_length=100, null=True, blank=True)
export_provider_type = models.ForeignKey(
DataProviderType, verbose_name="Service Type", null=True, on_delete=models.CASCADE
)
max_selection = models.DecimalField(
verbose_name="Max selection area",
default=250,
max_digits=12,
decimal_places=3,
help_text="This is the maximum area in square kilometers that can be exported "
"from this provider in a single DataPack.",
)
level_from = models.IntegerField(
verbose_name="Seed from level",
default=0,
null=True,
blank=True,
help_text="This determines the starting zoom level the tile export will seed from.",
)
level_to = models.IntegerField(
verbose_name="Seed to level",
default=10,
null=True,
blank=True,
help_text="This determines the highest zoom level the tile export will seed to.",
)
config = models.TextField(
default="",
null=True,
blank=True,
verbose_name="Configuration",
help_text="""WMS, TMS, WMTS, and ArcGIS-Raster require a MapProxy YAML configuration
with a Sources key of imagery and a Service Layer name of imagery; the validator also
requires a layers section, but this isn't used.
OSM Services also require a YAML configuration.""",
)
DATA_TYPES = [
(GeospatialDataType.VECTOR.value, ("Vector")),
(GeospatialDataType.RASTER.value, ("Raster")),
(GeospatialDataType.ELEVATION.value, ("Elevation")),
(GeospatialDataType.MESH.value, ("Mesh")),
(GeospatialDataType.POINT_CLOUD.value, ("Point Cloud")),
]
data_type = models.CharField(
choices=DATA_TYPES,
max_length=20,
verbose_name="Data Type",
null=True,
default="",
blank=True,
help_text="The type of data provided (e.g. elevation, raster, vector)",
)
user = models.ForeignKey(User, related_name="+", null=True, default=None, blank=True, on_delete=models.CASCADE)
license = models.ForeignKey(
License, related_name="+", null=True, blank=True, default=None, on_delete=models.CASCADE
)
zip = models.BooleanField(default=False)
display = models.BooleanField(default=False)
thumbnail = models.ForeignKey(
MapImageSnapshot,
blank=True,
null=True,
on_delete=models.SET_NULL,
help_text="A thumbnail image generated to give a high level" " preview of what a provider's data looks like.",
)
attribute_class = models.ForeignKey(
AttributeClass,
blank=True,
null=True,
on_delete=models.SET_NULL,
related_name="data_providers",
help_text="The attribute class is used to limit users access to resources using this data provider.",
)
the_geom = models.MultiPolygonField(
verbose_name="Covered Area",
srid=4326,
default="SRID=4326;MultiPolygon (((-180 -90,180 -90,180 90,-180 90,-180 -90)))",
)
# Used to store user list of user caches so that they can be invalidated.
provider_caches_key = "data_provider_caches"
class Meta: # pragma: no cover
managed = True
db_table = "export_provider"
# Check if config changed to updated geometry
__config = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__config = self.config
def update_geom(self):
from eventkit_cloud.tasks.helpers import download_data
from eventkit_cloud.ui.helpers import file_to_geojson
if self.config != self.__config:
orig_extent_url = load_provider_config(self.__config).get("extent_url")
config = load_provider_config(self.config)
extent_url = config.get("extent_url")
if extent_url and extent_url != orig_extent_url:
random_uuid = uuid.uuid4()
session = get_or_update_session(**config)
if not extent_url:
return
output_file = download_data(task_uid=str(random_uuid), input_url=extent_url, session=session)
geojson = file_to_geojson(output_file)
geometry = geojson.get("geometry") or geojson.get("features", [{}])[0].get("geometry")
if geometry:
self.the_geom = convert_polygon(GEOSGeometry(json.dumps(geometry), srid=4326))
def save(self, force_insert=False, force_update=False, *args, **kwargs):
# Something is closing the database connection which is raising an error.
# Using a separate process allows the connection to be closed in separate process while leaving it open.
proc = multiprocessing.dummy.Process(target=self.update_geom)
proc.start()
proc.join()
if not self.slug:
self.slug = self.name.replace(" ", "_").lower()
if len(self.slug) > 40:
self.slug = self.slug[0:39]
cache.delete(f"base-config-{self.slug}")
super(DataProvider, self).save(force_insert, force_update, *args, **kwargs)
def __str__(self):
return "{0}".format(self.name)
@property
def metadata(self):
from eventkit_cloud.utils.mapproxy import get_mapproxy_metadata_url
if not self.config:
return None
config = yaml.load(self.config)
url = config.get("sources", {}).get("info", {}).get("req", {}).get("url")
type = config.get("sources", {}).get("info", {}).get("type")
if url:
return {"url": get_mapproxy_metadata_url(self.slug), "type": type}
@property
def footprint_url(self):
from eventkit_cloud.utils.mapproxy import get_mapproxy_footprint_url
if not self.config:
return None
config = yaml.load(self.config)
url = config.get("sources", {}).get("footprint", {}).get("req", {}).get("url")
if url:
return get_mapproxy_footprint_url(self.slug)
@property
def layers(self) -> List[str]:
"""
Used to populate the list of vector layers, typically for contextual or styling information.
:return: A list of layer names.
"""
if self.data_type != GeospatialDataType.VECTOR.value:
return []
if not self.config:
# Often layer names are configured using an index number but this number is not very
# useful when using the data so fall back to the slug which should be more meaningful.
if not self.layer: # check for NoneType or empty string
return [self.slug]
try:
int(self.layer)
return [self.slug] # self.layer is an integer, so use the slug for better context.
except ValueError:
return [self.layer] # If we got here, layer is not None or an integer so use that.
config = clean_config(self.config, return_dict=True)
# As of EK 1.9.0 only vectors support multiple layers in a single provider
if self.export_provider_type.type_name in ["osm", "osm-generic"]:
return list(config.keys())
else:
return [layer.get("name") for layer in config.get("vector_layers", [])]
def get_use_bbox(self):
if self.export_provider_type is not None:
return self.export_provider_type.use_bbox
else:
return False
"""
Max datasize is the size in megabytes.
"""
@property
def max_data_size(self):
config = yaml.load(self.config)
return None if config is None else config.get("max_data_size", None)
def get_max_data_size(self, user=None):
if not user:
return self.max_data_size
# the usersizerule set is looped instead of using a queryset filter so that it can be prefetched.
if user:
user_size_rule = list(
filter(lambda user_size_rule: user_size_rule.user == user, self.usersizerule_set.all())
)
if user_size_rule:
return user_size_rule[0].max_data_size
return self.max_data_size
def get_max_selection_size(self, user=None):
if not user:
return self.max_selection
# the usersizerule set is looped instead of using a queryset filter so that it can be prefetched.
if user:
user_size_rule = list(
filter(lambda user_size_rule: user_size_rule.user == user, self.usersizerule_set.all())
)
if user_size_rule:
return user_size_rule[0].max_selection_size
return self.max_selection