def async_generate_tile( year: int, month: int, day: int, style_xml_template: str, zoom: int, x_pixel: float, y_pixel: float, osm_cato_path: str, cache_key: str, ) -> str: """ run celery background task to generate a mapnik tile & cache the tile :param year: request year as INT :param month: request month as INT :param day: request day as INT :param style_xml_template: path to style.xml :param zoom: mapnik zoom level :param x_pixel: mapnik x coordinate :param y_pixel: mapnik y coordinate :param osm_cato_path: path to osm cato :param cache_key: cache key for mapnik tile :return: """ # render requested tile tile: bytes = TileGenerator( request_date=date(year=int(year), month=int(month), day=int(day)), style_xml_template=style_xml_template, zoom=int(zoom), x_pixel=float(x_pixel), y_pixel=float(y_pixel), osm_cato_path=osm_cato_path, use_cache=True, ).render_tile() # create a md5 hash of the tile tile_hash: str = hashlib.md5(tile).hexdigest() # set url-tile cache content tile_cache: dict = {"process_id": None, "tile_hash": tile_hash} # update tile cache & url-tile cache content if zoom <= env.int("ZOOM_LEVEL", 13): # cache for ever cache.set(tile_hash, tile, None) cache.set(cache_key, tile_cache, None) else: # cache for time in TILE_CACHE_TIME cache.set(tile_hash, tile, env.int("TILE_CACHE_TIME", 2592000) * 10) cache.set(cache_key, tile_cache, env.int("TILE_CACHE_TIME", 2592000)) return tile_hash
from config.settings.base import * # noqa from config.settings.base import env # GENERAL # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key SECRET_KEY = env("DJANGO_SECRET_KEY") # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["example.com"]) # DATABASES # ------------------------------------------------------------------------------ DATABASES["default"] = env.db("DATABASE_URL") # noqa F405 DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa F405 DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) # noqa F405 # CACHES # ------------------------------------------------------------------------------ CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", "LOCATION": env("REDIS_URL"), "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", # Mimicing memcache behavior. # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior "IGNORE_EXCEPTIONS": True, }, } }
import hashlib from datetime import date from config.celery_app import app from config.settings.base import env from django.core.cache import cache from ohdm_django_mapnik.ohdm.tile import TileGenerator @app.task( soft_time_limit=env.int("TILE_GENERATOR_SOFT_TIMEOUT", 240), time_limit=env.int("TILE_GENERATOR_HARD_TIMEOUT", 360), ) def async_generate_tile( year: int, month: int, day: int, style_xml_template: str, zoom: int, x_pixel: float, y_pixel: float, osm_cato_path: str, cache_key: str, ) -> str: """ run celery background task to generate a mapnik tile & cache the tile :param year: request year as INT :param month: request month as INT :param day: request day as INT :param style_xml_template: path to style.xml :param zoom: mapnik zoom level
CACHE_MIDDLEWARE_ALIAS = 'default' # TODO-NORMAL: Update to 7 days once working properly CACHE_MIDDLEWARE_SECONDS = 3600 CACHE_MIDDLEWARE_KEY_PREFIX = '' # Security config if env.bool("MIMIC_PRODUCTION_LOCALLY"): pass else: CSRF_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True SECURE_SSL_REDIRECT = True SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') SECURE_HSTS_INCLUDE_SUBDOMAINS = True # TODO-NORMAL: Update to 7 days or 30 days once working properly SECURE_HSTS_SECONDS = env.int("DJANGO_SECURE_HSTS_SECONDS", default=60) SECURE_HSTS_PRELOAD = env.bool("DJANGO_SECURE_HSTS_PRELOAD", default=True) SECURE_CONTENT_TYPE_NOSNIFF = True # Admin Honeypot config ADMIN_HONEYPOT_EMAIL_ADMINS = True # Anymail # TODO-NORMAL: Remove console backend and switch to Twilio Sendgrid # EMAIL_BACKEND = "anymail.backends.sendgrid.EmailBackend" # ANYMAIL = { # "SENDGRID_API_KEY": env("SENDGRID_API_KEY"), # "SENDGRID_GENERATE_MESSAGE_ID": env("SENDGRID_GENERATE_MESSAGE_ID"), # "SENDGRID_MERGE_FIELD_FORMAT": env("SENDGRID_MERGE_FIELD_FORMAT"), # "SENDGRID_API_URL": env("SENDGRID_API_URL", default="https://api.sendgrid.com/v3/"), # }
from config.settings.base import * from config.settings.base import env DEBUG = False SECRET_KEY = env('SECRET_KEY', default='c#-!x^di-(n7@h@_2p@j(%^ce#8-@m=ager%x_zfqq%034qfdb') ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['localhost', '127.0.0.1', '.niangular.com']) # Database #DATABASES = {} #DATABASES['default'] = env.db('DATABASE_URL') DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql', 'NAME': env('DB_NAME'), 'USER': env('DB_USER'), 'PASSWORD': env('DB_PASSWORD'), 'HOST': '127.0.0.1', 'PORT': '5432', 'ATOMIC_REQUESTS': True, 'CONN_MAX_AGE': env.int('CONN_MAX_AGE', default=60) } }
def generate_tile(request, year: int, month: int, day: int, zoom: int, x_pixel: float, y_pixel: float) -> HttpResponse: """ get a mapnik tile, get it from cache if exist else it will be generated as a celery task :param request: django request :param year: request year as INT :param month: request month as INT :param day: request day as INT :param zoom: mapnik zoom level :param x_pixel: mapnik x coordinate :param y_pixel: mapnik y coordinate :return: """ # set tile cache key, where the celery task id & tile cache id is stored tile_cache_key: str = "{}-{}-{}-{}-{}-{}".format( int(year), int(month), int(day), int(zoom), int(x_pixel), int(y_pixel), ) # tile static typing tile: Optional[bytes] tile_process: AsyncResult # get tile cache tile_cache: Optional[dict] = cache.get(tile_cache_key, { "process_id": None, "tile_hash": None }) # check if process is running and wait for end if tile_cache: if tile_cache["process_id"]: tile_process = AsyncResult(tile_cache["process_id"]) for _ in range(0, env.int("TILE_GENERATOR_HARD_TIMEOUT", 360) * 2): sleep(0.5) tile_cache = cache.get(tile_cache_key, { "process_id": None, "tile_hash": None }) if tile_cache: if tile_cache["tile_hash"]: break # try get tile png & return it if tile_cache: if tile_cache["tile_hash"]: tile = cache.get(tile_cache["tile_hash"]) if tile: return HttpResponse(tile, content_type="image/jpeg") # if there is no tile process & no tile in cache, create one tile_process = async_generate_tile.delay( year=int(year), month=int(month), day=int(day), style_xml_template=OSM_CARTO_STYLE_XML, zoom=int(zoom), x_pixel=float(x_pixel), y_pixel=float(y_pixel), osm_cato_path=env("CARTO_STYLE_PATH"), cache_key=tile_cache_key, ) if not tile_cache: tile_cache = {"process_id": None, "tile_hash": None} tile_cache["process_id"] = tile_process.id # update cache if zoom <= env.int("ZOOM_LEVEL", 13): cache.set(tile_cache_key, tile_cache, None) else: cache.set(tile_cache_key, tile_cache, env.int("TILE_CACHE_TIME", 2592000)) try: tile_process.wait(timeout=env.int("TILE_GENERATOR_HARD_TIMEOUT", 360)) except exceptions.TimeoutError: return HttpResponse("Timeout when creating tile", status=500) except CoordinateOutOfRange as e: return HttpResponse(e, status=405) tile_cache["tile_hash"] = tile_process.get() tile = cache.get(tile_cache["tile_hash"]) if tile: return HttpResponse(tile, content_type="image/jpeg") return HttpResponse("Caching Error", status=500)