def _make_ini(root, base_dir):
ini = ''
paths = []
s = root.application.var('server.qgis.searchPathsForSVG')
if s:
paths.extend(s)
paths.extend(SVG_SEARCH_PATHS)
ini += f'''
[svg]
searchPathsForSVG={','.join(paths)}
'''
# set the cache dir and size=4096Kb
gws.ensure_dir('netcache', base_dir)
ini += fr'''
[cache]
directory={base_dir}/netcache
size=@Variant(\0\0\0\x81\0\0\0\0\0@\0\0)
'''
proxy = os.getenv('HTTPS_PROXY') or os.getenv('HTTP_PROXY')
if proxy:
p = gws.lib.net.parse_url(proxy)
ini += f'''
[proxy]
proxyEnabled=true
proxyType=HttpProxy
proxyHost={p.hostname}
proxyPort={p.port}
proxyUser={p.username}
proxyPassword={p.password}
'''
return '\n'.join(x.strip() for x in ini.splitlines())
def api_write(self, req: gws.IWebRequest, p: WriteParams) -> WriteResponse:
"""Write data to a new or existing file."""
dp, fname = self._check_file_path(p.path)
path = dp + '/' + fname
meta = self._read_metadata(path)
gws.ensure_dir(dp)
if not meta:
meta = {
'created_by': req.user.fid,
'created_time': gws.lib.date.now(),
}
if meta.get('deleted'):
self._unlink(path)
meta['updated_by'] = req.user.fid
meta['updated_time'] = gws.lib.date.now()
meta['deleted'] = False
gws.write_file_b(path, p.data)
self._write_metadata(path, meta)
return WriteResponse()
def __init__(self):
self.c = 0
gws.ensure_dir(TMP_DIR)
self.services = {
'wms': {
'image_formats': ['image/png'],
'max_output_pixels': [9000, 9000]
},
'wmts': {
'kvp': True,
'restful': False
}
}
self.globals = {
# https://mapproxy.org/docs/1.11.0/configuration.html#id14
# "By default MapProxy assumes lat/long (north/east) order for all geographic and x/y (east/north) order for all projected SRS."
# we need to change that because our extents are always x/y (lon/lat) even if a CRS says otherwise
'srs': {
'axis_order_en': ['EPSG:4326']
},
'cache': {
'base_dir': gws.MAPPROXY_CACHE_DIR,
'lock_dir': TMP_DIR + '/locks_' + gws.random_string(16),
'tile_lock_dir': TMP_DIR + '/tile_locks_' + gws.random_string(16),
'concurrent_tile_creators': 1,
'max_tile_limit': 5000,
},
'image': {
'resampling_method': 'bicubic',
'stretch_factor': 1.15,
'max_shrink_factor': 4.0,
'formats': {
'png8': {
'format': 'image/png',
'mode': 'P',
'colors': 256,
'transparent': True,
'resampling_method': 'bicubic',
},
'png24': {
'format': 'image/png',
'mode': 'RGBA',
'colors': 0,
'transparent': True,
'resampling_method': 'bicubic',
}
}
}
}
self.cfg = {}
def _reload_uwsgi(module):
pid_dir = gws.ensure_dir('pids', gws.TMP_DIR)
pattern = f'({module}).uwsgi.pid'
for p in gws.lib.os2.find_files(pid_dir, pattern):
gws.log.info(f'reloading {p}...')
gws.lib.os2.run(['/usr/local/bin/uwsgi', '--reload', p])
def run(action,
src_path: str,
replace: bool,
au_uid: str = None,
job: gws.lib.job.Job = None) -> Stats:
""""Import bplan data from a file or a directory."""
tmp_dir = None
if os2.is_file(src_path):
# a file is given - unpack it into a temp dir
tmp_dir = gws.ensure_dir(gws.TMP_DIR + '/bplan_' +
gws.random_string(32))
_extract(src_path, tmp_dir)
stats = None
try:
stats = _run2(action, tmp_dir or src_path, replace, au_uid, job)
except gws.lib.job.PrematureTermination as e:
pass
if tmp_dir:
shutil.rmtree(tmp_dir)
return stats
def configure(self):
self.root_dir = gws.ensure_dir(self.var('root'))
self.trash_dir = gws.ensure_dir(self.root_dir + '/' + TRASH_NAME)
self.db_path = self.root_dir + '/' + DB_NAME
with self._connect() as conn:
conn.execute('''CREATE TABLE IF NOT EXISTS meta(
path TEXT,
created_by TEXT,
created_time DATETIME,
updated_by TEXT,
updated_time DATETIME,
deleted INTEGER,
PRIMARY KEY (path)
) WITHOUT ROWID''')
os.chown(self.db_path, gws.UID, gws.GID)
def start_configured():
for p in gws.lib.os2.find_files(gws.SERVER_DIR, '.*'):
gws.lib.os2.unlink(p)
pid_dir = gws.ensure_dir('pids', gws.TMP_DIR)
commands = ini.create(gws.config.root(), gws.SERVER_DIR, pid_dir)
with open(_START_SCRIPT, 'wt') as fp:
fp.write('echo "----------------------------------------------------------"\n')
fp.write('echo "SERVER START"\n')
fp.write('echo "----------------------------------------------------------"\n')
fp.write('\n'.join(commands))
return _START_SCRIPT
def configure(self):
self.crs = self.var('crs')
self.db = t.cast(
gws.ext.db.provider.postgres.Object,
gws.base.db.require_provider(self, 'gws.ext.db.provider.postgres'))
self.templates: t.List[gws.ITemplate] = gws.base.template.bundle(self, self.var('templates'))
self.qgis_template: gws.ITemplate = gws.base.template.find(self.templates, subject='bplan.qgis')
self.info_template: gws.ITemplate = gws.base.template.find(self.templates, subject='bplan.info')
self.plan_table = self.db.configure_table(self.var('planTable'))
self.meta_table = self.db.configure_table(self.var('metaTable'))
self.data_dir = self.var('dataDir')
self.au_list = self.var('administrativeUnits')
self.type_list = self.var('planTypes')
self.image_quality = self.var('imageQuality')
p = self.var('exportDataModel')
self.export_data_model: t.Optional[gws.IDataModel] = self.create_child('gws.base.model', p) if p else None
for sub in 'png', 'pdf', 'cnv', 'qgs':
gws.ensure_dir(self.data_dir + '/' + sub)
self.key_col = 'plan_id'
self.au_key_col = 'ags'
self.au_name_col = 'gemeinde'
self.type_col = 'typ'
self.time_col = 'rechtskr'
self.x_coord_col = 'utm_ost'
self.y_coord_col = 'utm_nord'
gws.write_file(_RELOAD_FILE, gws.random_string(16))
self.root.application.monitor.add_path(_RELOAD_FILE)
def environ(root: gws.IRoot):
base_dir = gws.ensure_dir(gws.TMP_DIR + '/qqq')
# it's all a bit blurry, but the server appears to read 'ini' from OPTIONS_DIR
# while the app uses a profile
# NB: for some reason, the profile path will be profiles/profiles/default (sic!)
gws.ensure_dir('profiles', base_dir)
gws.ensure_dir('profiles/default', base_dir)
gws.ensure_dir('profiles/default/QGIS', base_dir)
gws.ensure_dir('profiles/profiles', base_dir)
gws.ensure_dir('profiles/profiles/default', base_dir)
gws.ensure_dir('profiles/profiles/default/QGIS', base_dir)
ini = _make_ini(root, base_dir)
gws.write_file(base_dir + '/profiles/default/QGIS/QGIS3.ini', ini)
gws.write_file(base_dir + '/profiles/profiles/default/QGIS/QGIS3.ini', ini)
# server options, as documented on
# see https://docs.qgis.org/testing/en/docs/user_manual/working_with_ogc/server/config.html#environment-variables
server_env = {
# not used here 'QGIS_PLUGINPATH': '',
# not used here 'QGIS_SERVER_LOG_FILE': '',
# see https://github.com/qgis/QGIS/pull/35738
'QGIS_SERVER_IGNORE_BAD_LAYERS': 'true',
'MAX_CACHE_LAYERS': root.application.var('server.qgis.maxCacheLayers'),
'QGIS_OPTIONS_PATH': base_dir + '/profiles/profiles/default',
'QGIS_SERVER_CACHE_DIRECTORY': gws.ensure_dir('servercache', base_dir),
'QGIS_SERVER_CACHE_SIZE': root.application.var('server.qgis.serverCacheSize'),
'QGIS_SERVER_LOG_LEVEL': root.application.var('server.qgis.serverLogLevel'),
# 'QGIS_SERVER_MAX_THREADS': 4,
# 'QGIS_SERVER_PARALLEL_RENDERING': 'false',
}
# qgis app options, mostly undocumented
qgis_env = {
'QGIS_PREFIX_PATH': '/usr',
'QGIS_DEBUG': root.application.var('server.qgis.debug'),
# 'QGIS_GLOBAL_SETTINGS_FILE': '/global_settings.ini',
'QGIS_CUSTOM_CONFIG_PATH': base_dir
}
# finally, there are lots of GDAL settings, some of those seem relevant
# http://trac.osgeo.org/gdal/wiki/ConfigOptions
gdal_env = {
'GDAL_FIX_ESRI_WKT': 'GEOGCS',
'GDAL_DEFAULT_WMS_CACHE_PATH': gws.ensure_dir('gdalcache', base_dir),
}
return gws.merge(
server_env,
qgis_env,
gdal_env,
)
def write_file(path, text):
pp = gws.lib.os2.parse_path(path)
if pp['dirname'].startswith(TEMP_DIR):
gws.ensure_dir(pp['dirname'])
with open(path, 'wt', encoding='utf8') as fp:
fp.write(text)
def create(root: gws.IRoot, base_dir, pid_dir):
def _write(p, s):
p = base_dir + '/' + p
s = '\n'.join(x.strip() for x in s.strip().splitlines())
with open(p, 'wt') as fp:
fp.write(s + '\n')
return p
for p in gws.lib.os2.find_files(base_dir, '(conf|ini)$'):
gws.lib.os2.unlink(p)
commands = []
frontends = []
# Check the marker file created by our docker build (see install/build.py)
try:
in_container = os.path.isfile('/.GWS_IN_CONTAINER')
except:
in_container = False
rsyslogd_enabled = in_container
# NB it should be possible to have QGIS running somewhere else
# so, if 'host' is not localhost, don't start QGIS here
qgis_enabled = root.application.var(
'server.qgis.enabled') and root.application.var(
'server.qgis.host') == 'localhost'
qgis_port = root.application.var('server.qgis.port')
qgis_workers = root.application.var('server.qgis.workers')
qgis_threads = root.application.var('server.qgis.threads')
qgis_socket = gws.TMP_DIR + '/uwsgi.qgis.sock'
web_enabled = root.application.var('server.web.enabled')
web_workers = root.application.var('server.web.workers')
web_threads = root.application.var('server.web.threads')
web_socket = gws.TMP_DIR + '/uwsgi.web.sock'
spool_enabled = root.application.var('server.spool.enabled')
spool_workers = root.application.var('server.spool.workers')
spool_threads = root.application.var('server.spool.threads')
spool_socket = gws.TMP_DIR + '/uwsgi.spooler.sock'
spool_dir = gws.SPOOL_DIR
spool_freq = root.application.var('server.spool.jobFrequency')
mapproxy_enabled = root.application.var(
'server.mapproxy.enabled') and os.path.exists(
gws.gis.mpx.config.CONFIG_PATH)
mapproxy_port = root.application.var('server.mapproxy.port')
mapproxy_workers = root.application.var('server.mapproxy.workers')
mapproxy_threads = root.application.var('server.mapproxy.threads')
mapproxy_socket = gws.TMP_DIR + '/uwsgi.mapproxy.sock'
log = root.application.var('server.log.path') or (
'syslog' if in_container else gws.LOG_DIR + '/gws.log')
nginx_log_level = 'info'
if root.application.developer_option('nginx.log_level_debug'):
nginx_log_level = 'debug'
nginx_rewrite_log = 'off'
if root.application.developer_option('nginx.rewrite_log_on'):
nginx_rewrite_log = 'on'
if log == 'syslog':
nginx_log = 'syslog:server=unix:/dev/log,nohostname,tag'
nginx_main_log = f'{nginx_log}=NGINX_MAIN'
nginx_qgis_log = f'{nginx_log}=NGINX_QGIS'
nginx_web_log = f'{nginx_log}=NGINX_WEB'
uwsgi_qgis_log = 'daemonize=true\nlogger=syslog:QGIS,local6'
uwsgi_web_log = 'daemonize=true\nlogger=syslog:WEB,local6'
uwsgi_mapproxy_log = 'daemonize=true\nlogger=syslog:MAPPROXY,local6'
uwsgi_spool_log = 'daemonize=true\nlogger=syslog:SPOOL,local6'
else:
nginx_main_log = nginx_qgis_log = nginx_web_log = log
uwsgi_qgis_log = uwsgi_web_log = uwsgi_mapproxy_log = uwsgi_spool_log = f'daemonize={log}'
# be rude and reload 'em as fast as possible
mercy = 5
# @TODO: do we need more granular timeout configuration?
DEFAULT_BASE_TIMEOUT = 60
DEFAULT_SPOOL_TIMEOUT = 300
base_timeout = int(
root.application.var('server.timeout', default=DEFAULT_BASE_TIMEOUT))
qgis_timeout = base_timeout + 10
qgis_front_timeout = qgis_timeout + 10
mapproxy_timeout = qgis_front_timeout + 10
web_timeout = mapproxy_timeout + 10
web_front_timeout = web_timeout + 10
spool_timeout = int(
root.application.var('server.spool.timeout',
default=DEFAULT_SPOOL_TIMEOUT))
gws.log.debug(
f'TIMEOUTS: {[qgis_timeout, qgis_front_timeout, mapproxy_timeout, web_timeout, web_front_timeout, spool_timeout]}'
)
stdenv = '\n'.join(f'env = {k}={v}' for k, v in os.environ.items()
if k.startswith('GWS_'))
stdenv += f'\nTMP={gws.TMP_DIR}'
stdenv += f'\nTEMP={gws.TMP_DIR}'
# rsyslogd
# ---------------------------------------------------------
if rsyslogd_enabled:
# based on /etc/rsyslog.conf
syslog_conf = f"""
##
module(
load="imuxsock"
SysSock.UsePIDFromSystem="on"
)
module(
load="imklog"
PermitNonKernelFacility="on"
)
template(name="gws" type="list") {{
property(name="timestamp" dateFormat="mysql")
constant(value=" ")
property(name="syslogtag")
constant(value=" ")
property(name="msg" spifno1stsp="on")
property(name="msg" droplastlf="on")
constant(value="\\n")
}}
module(
load="builtin:omfile"
Template="gws"
)
# *.*;kern.none /dev/stdout
# kern.* -/var/log/kern.log
*.* /dev/stdout
"""
path = _write('syslog.conf', syslog_conf)
commands.append(f'rsyslogd -i {pid_dir}/rsyslogd.pid -f {path}')
# qgis
# ---------------------------------------------------------
if qgis_enabled:
qgis_server = gws.import_from_path('gws/plugin/qgis/server.py')
# partially inspired by
# https://github.com/elpaso/qgis2-server-vagrant/blob/master/docs/index.rst
# harakiri doesn't seem to work with worker-exec
# besides, it's a bad idea anyways, because killing them prematurely
# doesn't give them a chance to fully preload a project
srv = qgis_server.EXEC_PATH
ini = f"""
[uwsgi]
uid = {gws.UID}
gid = {gws.GID}
chmod-socket = 777
fastcgi-socket = {qgis_socket}
{uwsgi_qgis_log}
master = true
pidfile = {pid_dir}/qgis.uwsgi.pid
processes = {qgis_workers}
reload-mercy = {mercy}
threads = {qgis_threads}
vacuum = true
worker-exec = {srv}
worker-reload-mercy = {mercy}
{stdenv}
"""
for k, v in qgis_server.environ(root).items():
ini += f'env = {k}={v}\n'
path = _write('uwsgi_qgis.ini', ini)
commands.append(f'uwsgi --ini {path}')
frontends.append(f"""
server {{
listen {qgis_port};
server_name qgis;
error_log {nginx_qgis_log} {nginx_log_level};
access_log {nginx_qgis_log};
rewrite_log {nginx_rewrite_log};
location / {{
gzip off;
fastcgi_pass unix:{qgis_socket};
fastcgi_read_timeout {qgis_front_timeout};
# add_header 'Access-Control-Allow-Origin' *;
# add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
# replace mapproxy forward params (e.g. LAYERS__gws) with their real names
if ($args ~* (.*?)(layers=-)(.*)) {{
set $args $1$3;
}}
if ($args ~ (.*?)(__gws)(.*)) {{
set $args $1$3;
}}
include /etc/nginx/fastcgi_params;
}}
}}
""")
# web
# ---------------------------------------------------------
if web_enabled:
ini = f"""
[uwsgi]
uid = {gws.UID}
gid = {gws.GID}
buffer-size = 65535
chmod-socket = 777
die-on-term = true
harakiri = {web_timeout}
harakiri-verbose = true
{uwsgi_web_log}
pidfile = {pid_dir}/web.uwsgi.pid
post-buffering = 65535
processes = {web_workers}
pythonpath = {gws.APP_DIR}
reload-mercy = {mercy}
spooler-external = {spool_dir}
threads = {web_threads}
uwsgi-socket = {web_socket}
vacuum = true
worker-reload-mercy = {mercy}
wsgi-file = {gws.APP_DIR}/gws/base/web/web_app.py
{stdenv}
"""
path = _write('uwsgi_web.ini', ini)
commands.append(f'uwsgi --ini {path}')
roots = ''
rewr = ''
app = gws.config.root().application
for s in app.web_sites:
site = t.cast(gws.base.web.site.Object, s)
for r in site.rewrite_rules:
rewr += f'rewrite {r.pattern} {r.target} last;\n'
d = site.static_root.dir
roots += f"""
location =/ {{
root {d};
index index.html;
}}
location / {{
root {d};
try_files $uri @cache;
}}
"""
# @TODO multisites
break
# this is in MB
max_body_size = int(
root.application.var('server.web.maxRequestLength', default=1))
client_buffer_size = 4 # MB
client_tmp_dir = gws.ensure_dir(gws.TMP_DIR + '/nginx')
web_common = f"""
error_log {nginx_web_log} {nginx_log_level};
access_log {nginx_web_log} apm;
rewrite_log {nginx_rewrite_log};
client_max_body_size {max_body_size}m;
client_body_buffer_size {client_buffer_size}m;
client_body_temp_path {client_tmp_dir};
# @TODO: optimize, disallow _ rewriting
{rewr}
{roots}
location @cache {{
root {gws.WEB_CACHE_DIR};
try_files $uri @app;
}}
location @app {{
uwsgi_pass unix://{web_socket};
uwsgi_read_timeout {web_front_timeout};
{_uwsgi_params}
}}
"""
ssl_crt = root.application.var('web.ssl.crt')
ssl_key = root.application.var('web.ssl.key')
ssl_hsts = ''
s = root.application.var('web.ssl.hsts')
if s:
ssl_hsts = f'add_header Strict-Transport-Security "max-age={s}; includeSubdomains";'
# NB don't include xml (some WMS clients don't understand gzip)
# text/xml application/xml application/xml+rss
gzip = """
gzip on;
gzip_types text/plain text/css application/json application/javascript text/javascript;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
"""
if ssl_crt:
frontends.append(f"""
server {{
listen 80 default_server;
server_name gws;
return 301 https://$host$request_uri;
}}
server {{
listen 443 ssl default_server;
server_name gws;
ssl_certificate {ssl_crt};
ssl_certificate_key {ssl_key};
{ssl_hsts}
{gzip}
{web_common}
}}
""")
else:
frontends.append(f"""
server {{
listen 80 default_server;
server_name gws;
{gzip}
{web_common}
}}
""")
# mapproxy
# ---------------------------------------------------------
# see https://github.com/mapproxy/mapproxy/issues/282 about 'wsgi-disable-file-wrapper'
if mapproxy_enabled:
ini = f"""
[uwsgi]
uid = {gws.UID}
gid = {gws.GID}
chmod-socket = 777
die-on-term = true
harakiri = {mapproxy_timeout}
harakiri-verbose = true
http = :{mapproxy_port}
http-to = {mapproxy_socket}
{uwsgi_mapproxy_log}
pidfile = {pid_dir}/mapproxy.uwsgi.pid
post-buffering = 65535
processes = {mapproxy_workers}
pythonpath = {gws.APP_DIR}
reload-mercy = {mercy}
threads = {mapproxy_threads}
uwsgi-socket = {mapproxy_socket}
vacuum = true
worker-reload-mercy = {mercy}
wsgi-disable-file-wrapper = true
wsgi-file = {gws.APP_DIR}/gws/gis/mpx/mpx_app.py
{stdenv}
"""
path = _write('uwsgi_mapproxy.ini', ini)
commands.append(f'uwsgi --ini {path}')
# spooler
# ---------------------------------------------------------
if spool_enabled:
ini = f"""
[uwsgi]
uid = {gws.UID}
gid = {gws.GID}
chmod-socket = 777
die-on-term = true
harakiri = {spool_timeout}
harakiri-verbose = true
{uwsgi_spool_log}
master = true
pidfile = {pid_dir}/spool.uwsgi.pid
post-buffering = 65535
processes = {spool_workers}
pythonpath = {gws.APP_DIR}
reload-mercy = {mercy}
spooler = {spool_dir}
spooler-frequency = {spool_freq}
threads = {spool_threads}
uwsgi-socket = {spool_socket}
vacuum = true
worker-reload-mercy = {mercy}
wsgi-file = {gws.APP_DIR}/gws/server/spool_app.py
{stdenv}
"""
path = _write('uwsgi_spool.ini', ini)
commands.append(f'uwsgi --ini {path}')
# main
# ---------------------------------------------------------
frontends_str = '\n\n'.join(frontends)
# log_format: https://www.nginx.com/blog/using-nginx-logging-for-application-performance-monitoring/
u = pwd.getpwuid(gws.UID).pw_name
g = grp.getgrgid(gws.GID).gr_name
daemon = 'daemon off;' if in_container else ''
nginx_conf = f"""
worker_processes auto;
pid {pid_dir}/nginx.pid;
user {u} {g};
events {{
worker_connections 768;
# multi_accept on;
}}
{daemon}
error_log {nginx_main_log} {nginx_log_level};
http {{
log_format apm '$remote_addr'
' method=$request_method request="$request"'
' request_length=$request_length'
' status=$status bytes_sent=$bytes_sent'
' referer=$http_referer'
' user_agent="$http_user_agent"'
' request_time=$request_time'
' upstream_response_time=$upstream_response_time'
' upstream_connect_time=$upstream_connect_time'
' upstream_header_time=$upstream_header_time';
access_log {nginx_main_log};
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
{frontends_str}
}}
"""
path = _write('nginx.conf', nginx_conf)
commands.append(f'nginx -c {path}')
return commands
