class TileServer: def __init__(self): self.path_regex = re.compile(r'^/(\d+)/(-?\d+)/(-?\d+)/(-?\d+).png$') self.cookie_regex = re.compile(r'(^| )c3nav_tile_access="?([^;" ]+)"?') try: self.upstream_base = os.environ['C3NAV_UPSTREAM_BASE'].strip('/') except KeyError: raise Exception('C3NAV_UPSTREAM_BASE needs to be set.') try: self.data_dir = os.environ.get('C3NAV_DATA_DIR', 'data') except KeyError: raise Exception('C3NAV_DATA_DIR needs to be set.') if not os.path.exists(self.data_dir): os.mkdir(self.data_dir) self.tile_secret = os.environ.get('C3NAV_TILE_SECRET', None) if not self.tile_secret: tile_secret_file = None try: tile_secret_file = os.environ['C3NAV_TILE_SECRET_FILE'] self.tile_secret = open(tile_secret_file).read().strip() except KeyError: raise Exception( 'C3NAV_TILE_SECRET or C3NAV_TILE_SECRET_FILE need to be set.' ) except FileNotFoundError: raise Exception( 'The C3NAV_TILE_SECRET_FILE (%s) does not exist.' % tile_secret_file) self.reload_interval = int(os.environ.get('C3NAV_RELOAD_INTERVAL', 60)) self.http_auth = os.environ.get('C3NAV_HTTP_AUTH', None) if self.http_auth: self.http_auth = HTTPBasicAuth(*self.http_auth.split(':', 1)) self.auth_headers = { 'X-Tile-Secret': base64.b64encode(self.tile_secret.encode()).decode() } self.cache_package = None self.cache_package_etag = None self.cache_package_filename = None cache = self.get_cache_client() wait = 1 while True: success = self.load_cache_package(cache=cache) if success: logger.info('Cache package successfully loaded.') break logger.info('Retrying after %s seconds...' % wait) time.sleep(wait) wait = min(10, wait * 2) threading.Thread(target=self.update_cache_package_thread, daemon=True).start() @staticmethod def get_cache_client(): return pylibmc.Client(["127.0.0.1"], binary=True, behaviors={ "tcp_nodelay": True, "ketama": True }) def update_cache_package_thread(self): cache = self.get_cache_client() # different thread → different client! while True: time.sleep(self.reload_interval) self.load_cache_package(cache=cache) def get_date_header(self): return 'Date', formatdate(timeval=time.time(), localtime=False, usegmt=True) def load_cache_package(self, cache): logger.debug('Downloading cache package from upstream...') try: headers = self.auth_headers.copy() if self.cache_package_etag is not None: headers['If-None-Match'] = self.cache_package_etag r = requests.get(self.upstream_base + '/map/cache/package.tar.xz', headers=headers, auth=self.http_auth) if r.status_code == 403: logger.error( 'Rejected cache package download with Error 403. Tile secret is probably incorrect.' ) return False if r.status_code == 401: logger.error( 'Rejected cache package download with Error 401. You have HTTP Auth active.' ) return False if r.status_code == 304: if self.cache_package is not None: logger.debug('Not modified.') cache[ 'cache_package_filename'] = self.cache_package_filename return True logger.error('Unexpected not modified.') return False r.raise_for_status() except Exception as e: logger.error('Cache package download failed: %s' % e) return False logger.debug('Recieving and loading new cache package...') try: self.cache_package = CachePackage.read(BytesIO(r.content)) self.cache_package_etag = r.headers.get('ETag', None) except Exception as e: logger.error('Cache package parsing failed: %s' % e) return False try: self.cache_package_filename = os.path.join( self.data_dir, datetime.now().strftime('%Y-%m-%d_%H-%M-%S-%f') + '.pickle') with open(self.cache_package_filename, 'wb') as f: pickle.dump(self.cache_package, f) cache.set('cache_package_filename', self.cache_package_filename) except Exception as e: self.cache_package_etag = None logger.error('Saving pickled package failed: %s' % e) return False return True def not_found(self, start_response, text): start_response('404 Not Found', [ self.get_date_header(), ('Content-Type', 'text/plain'), ('Content-Length', str(len(text))) ]) return [text] def internal_server_error(self, start_response, text=b'internal server error'): start_response('500 Internal Server Error', [ self.get_date_header(), ('Content-Type', 'text/plain'), ('Content-Length', str(len(text))) ]) return [text] def deliver_tile(self, start_response, etag, data): start_response('200 OK', [ self.get_date_header(), ('Content-Type', 'image/png'), ('Content-Length', str(len(data))), ('Cache-Control', 'no-cache'), ('ETag', etag) ]) return [data] def get_cache_package(self): try: cache_package_filename = self.cache.get('cache_package_filename') except pylibmc.Error as e: logger.warning('pylibmc error in get_cache_package(): %s' % e) cache_package_filename = None if cache_package_filename is None: logger.warning('cache_package_filename went missing.') return self.cache_package if self.cache_package_filename != cache_package_filename: logger.debug('Loading new cache package in worker.') self.cache_package_filename = cache_package_filename with open(self.cache_package_filename, 'rb') as f: self.cache_package = pickle.load(f) return self.cache_package cache_lock = multiprocessing.Lock() @property def cache(self): cache = self.get_cache_client() self.__dict__['cache'] = cache return cache def __call__(self, env, start_response): path_info = env['PATH_INFO'] match = self.path_regex.match(path_info) if match is None: return self.not_found(start_response, b'invalid tile path.') level, zoom, x, y = match.groups() zoom = int(zoom) if not (-2 <= zoom <= 5): return self.not_found(start_response, b'zoom out of bounds.') # do this to be thread safe try: cache_package = self.get_cache_package() except Exception as e: logger.error('get_cache_package() failed: %s' % e) return self.internal_server_error(start_response) # check if bounds are valid x = int(x) y = int(y) minx, miny, maxx, maxy = get_tile_bounds(zoom, x, y) if not cache_package.bounds_valid(minx, miny, maxx, maxy): return self.not_found(start_response, b'coordinates out of bounds.') # get level level = int(level) level_data = cache_package.levels.get(level) if level_data is None: return self.not_found(start_response, b'invalid level.') # build cache keys last_update = level_data.history.last_update(minx, miny, maxx, maxy) base_cache_key = build_base_cache_key(last_update) # decode access permissions access_permissions = set() access_cache_key = '0' cookie = env.get('HTTP_COOKIE', None) if cookie: cookie = self.cookie_regex.search(cookie) if cookie: cookie = cookie.group(2) access_permissions = ( parse_tile_access_cookie(cookie, self.tile_secret) & set(level_data.restrictions[minx:maxx, miny:maxy])) access_cache_key = build_access_cache_key(access_permissions) # check browser cache if_none_match = env.get('HTTP_IF_NONE_MATCH') tile_etag = build_tile_etag(level, zoom, x, y, base_cache_key, access_cache_key, self.tile_secret) if if_none_match == tile_etag: start_response('304 Not Modified', [ self.get_date_header(), ('Content-Length', '0'), ('ETag', tile_etag) ]) return [b''] cache_key = path_info + '_' + tile_etag cached_result = self.cache.get(cache_key) if cached_result is not None: return self.deliver_tile(start_response, tile_etag, cached_result) r = requests.get( '%s/map/%d/%d/%d/%d/%s.png' % (self.upstream_base, level, zoom, x, y, access_cache_key), headers=self.auth_headers, auth=self.http_auth) if r.status_code == 200 and r.headers['Content-Type'] == 'image/png': self.cache.set(cache_key, r.content) return self.deliver_tile(start_response, tile_etag, r.content) start_response('%d %s' % (r.status_code, r.reason), [ self.get_date_header(), ('Content-Length', str(len(r.content))), ('Content-Type', r.headers.get('Content-Type', 'text/plain')) ]) return [r.content]
def url_and_auth_from_admin_url(admin_url: str) -> Tuple[str, Dict]: url, auth, shop_name = split_admin_url(admin_url) auth = HTTPBasicAuth(*auth.split(":")) shop_url = f"https://{shop_name}.myshopify.com/admin/api/{API_VERSION}" return shop_url, auth