def get_info(component_path, format):
component_details = open(os.path.join(component_path, ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': component_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = None
try:
master_proc_pid = int(open(os.path.join(component_path, MISC.PIDFILE)).read())
except(IOError, ValueError):
# Ok, no such file or it's empty
pass
if master_proc_pid:
out['component_running'] = True
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid for elem in master_proc.children())
out['master_proc_connections'] = format_connections(master_proc.connections(), format)
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time()).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time(), UTC).isoformat()
out['master_proc_username'] = master_proc.username()
out['master_proc_name'] = master_proc.name()
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time()).isoformat()
out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time(), UTC).isoformat()
out['worker_{}_connections'.format(pid)] = format_connections(worker.connections(), format)
return out
def test_valid_is_running(self):
p = Process()
_, f = mkstemp()
try:
with open(f, 'w') as pid_file:
pid_file.write('{0} {1:6f}'.format(p.pid, p.create_time()))
self.assertTrue(_is_running(f))
finally:
unlink(f)
def test_valid_is_running(self):
p = Process()
d = mkdtemp()
f = _get_pid_filename(d)
try:
with open(f, 'w') as pid_file:
pid_file.write('{0} {1:6f}'.format(p.pid, p.create_time()))
self.assertTrue(_is_running(d))
finally:
rmtree(d)
def _set_running(filename):
"""Write the current process information to disk.
:param filename: The name of the file where the process information will be written.
"""
if isfile(str(filename)):
raise PidFileExistsError()
p = Process()
with open(filename, 'w') as f:
f.write('{0} {1:.6f}'.format(p.pid, p.create_time()))
def _set_running(directory):
"""Write the current process information to disk.
:param directory: The name of the directory where the process information will be written.
"""
filename = _get_pid_filename(directory)
if isfile(str(filename)):
raise PidFileExistsError
p = Process()
with open(filename, 'w') as f:
f.write('{0} {1:.6f}'.format(p.pid, p.create_time()))
def test_running_valid_file(self):
d = mkdtemp()
f = realpath(join(d, '.pid'))
try:
_set_running(f)
with open(f, 'r') as pid_file:
process_info = pid_file.read().split()
p = Process()
self.assertEquals(p.pid, int(process_info[0]))
self.assertEquals(p.create_time(), float(process_info[1]))
finally:
shutil.rmtree(d)
def test_running_valid_file(self):
temp_d = mkdtemp()
d = realpath(join(temp_d, '.pid'))
mkdir(d)
try:
_set_running(d)
with open(join(d, 'INFO'), 'r') as pid_file:
process_info = pid_file.read().split()
p = Process()
self.assertEquals(p.pid, int(process_info[0]))
self.assertEquals(p.create_time(), float(process_info[1]))
finally:
rmtree(temp_d)
def _is_running(pid_file):
""" Determine whether or not the process is currently running.
:param pid_file: The PID file containing the process information.
:return: Whether or not the process is currently running.
"""
if not isfile(str(pid_file)):
return False
pid, create_time = _read_pid_file(pid_file)
try:
current = Process(pid)
except NoSuchProcess:
return False
if current.create_time() == create_time:
return True
_delete(pid_file)
return False
def process_with_stats(name, stdin, times=1):
if times > 1:
results = []
for _ in xrange(0, times):
results.append(process_with_stats(name, stdin, 1))
result = {'execution': reduce(lambda a, c:
a+c['execution']/float(times),
results, 0),
'memory': reduce(lambda a, c:
max(a, c['memory']), results, 0),
'output': results[0]['output'],
'status': results[0]['status']}
return result
process = Popen(name, stdin=PIPE, stdout=PIPE, stderr=DEVNULL,
close_fds=True)
process.stdin.write(stdin.getvalue())
process.stdin.close()
stats = Process(process.pid)
memory_usage = stats.memory_info().rss
while process.poll() is None:
try:
memory_usage = max(memory_usage, stats.memory_info().rss)
except:
memory_usage = 0
sleep(1/1000.0)
execution_time = time() - stats.create_time()
output = process.stdout.read()
if memory_usage == 0 and process.returncode == 0:
return process_with_stats(name, stdin)
return {'execution': execution_time,
'memory': memory_usage,
'output': output,
'status': process.returncode}
def _is_running(directory):
""" Determine whether or not the process is currently running.
:param directory: The PID directory containing the process information.
:return: True if there is another process running, False if there is not.
"""
if not isdir(str(directory)):
return _is_locked(directory)
try:
pid, create_time = _read_pid_file(_get_pid_filename(directory))
except InvalidPidFileError:
return _is_locked(directory, True)
try:
current = Process(pid)
except NoSuchProcess:
return _is_locked(directory, True)
if current.create_time() != create_time:
return _is_locked(directory, True)
return True
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': abs_args_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = None
try:
master_proc_pid = int(open(os.path.join(abs_args_path, MISC.PIDFILE)).read())
except(IOError, ValueError):
# Ok, no such file or it's empty
pass
if master_proc_pid:
out['component_running'] = True
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid for elem in master_proc.children())
out['master_proc_connections'] = master_proc.connections()
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time()).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time(), UTC).isoformat()
out['master_proc_username'] = master_proc.username()
out['master_proc_name'] = master_proc.name()
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time()).isoformat()
out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time(), UTC).isoformat()
out['worker_{}_connections'.format(pid)] = worker.connections()
if getattr(args, 'json', False):
out['component_details'] = loads(out['component_details'])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(','))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(['t', 't'])
rows = [['Key', 'Value']]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
def _get_create_time(process: psutil.Process):
# create_time
try:
create_time = datetime.fromtimestamp(process.create_time())
except OSError:
create_time = datetime.fromtimestamp(process.boot_time())
def setup_pyramid(comp, config):
env = comp.env
is_debug = env.core.debug
# Session factory
config.set_session_factory(WebSession)
# Empty authorization policy. Why do we need this?
# NOTE: Authentication policy is set up in then authentication component!
authz_policy = ACLAuthorizationPolicy()
config.set_authorization_policy(authz_policy)
_setup_pyramid_debugtoolbar(comp, config)
_setup_pyramid_tm(comp, config)
_setup_pyramid_mako(comp, config)
# COMMON REQUEST'S ATTRIBUTES
config.add_request_method(lambda req: env, 'env', property=True)
config.add_request_method(
lambda req: req.path_info.lower().startswith('/api/'),
'is_api',
property=True)
# ERROR HANGLING
comp.error_handlers = list()
@comp.error_handlers.append
def api_error_handler(request, err_info, exc, exc_info):
if request.is_api or request.is_xhr:
return exception.json_error_response(request,
err_info,
exc,
exc_info,
debug=is_debug)
@comp.error_handlers.append
def html_error_handler(request, err_info, exc, exc_info):
return exception.html_error_response(request,
err_info,
exc,
exc_info,
debug=is_debug)
def error_handler(request, err_info, exc, exc_info, **kwargs):
for handler in comp.error_handlers:
result = handler(request, err_info, exc, exc_info)
if result is not None:
return result
config.registry.settings['error.err_response'] = error_handler
config.registry.settings['error.exc_response'] = error_handler
config.include(exception)
config.add_tween(
'nextgisweb.pyramid.util.header_encoding_tween_factory',
over=('nextgisweb.pyramid.exception.unhandled_exception_tween_factory',
))
# INTERNATIONALIZATION
# Substitute localizer from pyramid with our own, original is
# too tied to translationstring, that works strangely with string
# interpolation via % operator.
def localizer(request):
return request.env.core.localizer(request.locale_name)
config.add_request_method(localizer, 'localizer', property=True)
# Replace default locale negotiator with session-based one
def locale_negotiator(request):
return request.session.get('pyramid.locale', env.core.locale_default)
config.set_locale_negotiator(locale_negotiator)
# TODO: Need to get rid of translation dirs!
# Currently used only to search for jed-files.
from ..package import pkginfo as _pkginfo
for pkg in _pkginfo.packages:
dirname = resource_filename(pkg, 'locale')
if os.path.isdir(dirname):
config.add_translation_dirs(dirname)
# STATIC FILES
if is_debug:
# In debug build static_key from proccess startup time
rproc = Process(os.getpid())
# When running under control of uWSGI master process use master's startup time
if rproc.name() == 'uwsgi' and rproc.parent().name() == 'uwsgi':
rproc = rproc.parent()
_logger.debug("Found uWSGI master process PID=%d", rproc.pid)
# Use 4-byte hex representation of 1/5 second intervals
comp.static_key = '/' + hex(int(rproc.create_time() * 5) % (2 ** 64)) \
.replace('0x', '').replace('L', '')
_logger.debug("Using startup time static key [%s]",
comp.static_key[1:])
else:
# In production mode build static_key from pip freeze output
comp.static_key = '/' + pip_freeze()[0]
_logger.debug("Using pip freeze static key [%s]", comp.static_key[1:])
config.add_static_view('/static{}/asset'.format(comp.static_key),
'nextgisweb:static',
cache_max_age=3600)
config.add_route('amd_package', '/static{}/amd/*subpath'.format(comp.static_key)) \
.add_view(static_amd_file)
# Collect external AMD-packages from other components
amd_base = []
for c in comp._env.chain('amd_base'):
amd_base.extend(c.amd_base)
config.add_request_method(lambda r: amd_base,
'amd_base',
property=True,
reify=True)
# RENDERERS
config.add_renderer('json', json_renderer)
# Filter for quick translation. Defines function tr, which we can use
# instead of request.localizer.translate in mako templates.
def tr_subscriber(event):
event['tr'] = event['request'].localizer.translate
config.add_subscriber(tr_subscriber, BeforeRender)
# OTHERS
config.add_route('home', '/').add_view(home)
def ctpl(n):
return 'nextgisweb:pyramid/template/%s.mako' % n
config.add_route('pyramid.control_panel', '/control-panel', client=()) \
.add_view(control_panel, renderer=ctpl('control_panel'))
config.add_route('pyramid.favicon', '/favicon.ico').add_view(favicon)
config.add_route('pyramid.control_panel.pkginfo',
'/control-panel/pkginfo').add_view(
pkginfo, renderer=ctpl('pkginfo'))
config.add_route('pyramid.control_panel.backup.browse',
'/control-panel/backup/').add_view(
backup_browse, renderer=ctpl('backup'))
config.add_route(
'pyramid.control_panel.backup.download',
'/control-panel/backup/{filename}').add_view(backup_download)
config.add_route('pyramid.control_panel.cors',
'/control-panel/cors').add_view(cors,
renderer=ctpl('cors'))
config.add_route('pyramid.control_panel.custom_css',
'/control-panel/custom-css').add_view(
custom_css, renderer=ctpl('custom_css'))
config.add_route('pyramid.control_panel.logo',
'/control-panel/logo').add_view(cp_logo,
renderer=ctpl('logo'))
config.add_route('pyramid.control_panel.system_name',
'/control-panel/system-name').add_view(
system_name, renderer=ctpl('system_name'))
config.add_route('pyramid.control_panel.miscellaneous',
'/control-panel/miscellaneous').add_view(
miscellaneous, renderer=ctpl('miscellaneous'))
config.add_route('pyramid.control_panel.home_path',
'/control-panel/home_path').add_view(
home_path, renderer=ctpl('home_path'))
config.add_route('pyramid.locale', '/locale/{locale}').add_view(locale)
comp.test_request_handler = None
config.add_route('pyramid.test_request', '/test/request/') \
.add_view(test_request)
config.add_route('pyramid.test_exception_handled', '/test/exception/handled') \
.add_view(test_exception_handled)
config.add_route('pyramid.test_exception_unhandled', '/test/exception/unhandled') \
.add_view(test_exception_unhandled)
config.add_route('pyramid.test_timeout', '/test/timeout') \
.add_view(test_timeout)
comp.control_panel = dm.DynMenu(
dm.Label('info', _("Info")),
dm.Link(
'info/pkginfo', _("Package versions"), lambda args:
(args.request.route_url('pyramid.control_panel.pkginfo'))),
dm.Label('settings', _("Settings")),
dm.Link(
'settings/core', _("Web GIS name"), lambda args:
(args.request.route_url('pyramid.control_panel.system_name'))),
dm.Link(
'settings/cors', _("Cross-origin resource sharing (CORS)"),
lambda args:
(args.request.route_url('pyramid.control_panel.cors'))),
dm.Link(
'settings/custom_css', _("Custom CSS"), lambda args:
(args.request.route_url('pyramid.control_panel.custom_css'))),
dm.Link(
'settings/logo', _("Custom logo"), lambda args:
(args.request.route_url('pyramid.control_panel.logo'))),
dm.Link(
'settings/miscellaneous', _("Miscellaneous"), lambda args:
(args.request.route_url('pyramid.control_panel.miscellaneous'))),
dm.Link(
'settings/home_path', _("Home path"), lambda args:
(args.request.route_url('pyramid.control_panel.home_path'))),
)
if comp.options['backup.download']:
comp.control_panel.add(
dm.Link(
'info/backups', _("Backups"), lambda args: args.request.
route_url('pyramid.control_panel.backup.browse')))
