def get_info(component_path, format): component_details = open(os.path.join(component_path, ZATO_INFO_FILE)).read() out = { 'component_details': component_details, 'component_full_path': component_path, 'component_host': current_host(), 'component_running': False, 'current_time': datetime.now().isoformat(), 'current_time_utc': datetime.utcnow().isoformat(), 'master_proc_connections': None, 'master_proc_pid': None, 'master_proc_name': None, 'master_proc_create_time': None, 'master_proc_create_time_utc': None, 'master_proc_username': None, 'master_proc_workers_no': None, 'master_proc_workers_pids': None, } master_proc_pid = None try: master_proc_pid = int(open(os.path.join(component_path, MISC.PIDFILE)).read()) except(IOError, ValueError): # Ok, no such file or it's empty pass if master_proc_pid: out['component_running'] = True master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.children()) out['master_proc_connections'] = format_connections(master_proc.connections(), format) out['master_proc_pid'] = master_proc.pid out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time()).isoformat() out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time(), UTC).isoformat() out['master_proc_username'] = master_proc.username() out['master_proc_name'] = master_proc.name() out['master_proc_workers_no'] = len(workers_pids) out['master_proc_workers_pids'] = workers_pids for pid in workers_pids: worker = Process(pid) out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time()).isoformat() out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time(), UTC).isoformat() out['worker_{}_connections'.format(pid)] = format_connections(worker.connections(), format) return out
def test_valid_is_running(self): p = Process() _, f = mkstemp() try: with open(f, 'w') as pid_file: pid_file.write('{0} {1:6f}'.format(p.pid, p.create_time())) self.assertTrue(_is_running(f)) finally: unlink(f)
def test_valid_is_running(self): p = Process() d = mkdtemp() f = _get_pid_filename(d) try: with open(f, 'w') as pid_file: pid_file.write('{0} {1:6f}'.format(p.pid, p.create_time())) self.assertTrue(_is_running(d)) finally: rmtree(d)
def _set_running(filename): """Write the current process information to disk. :param filename: The name of the file where the process information will be written. """ if isfile(str(filename)): raise PidFileExistsError() p = Process() with open(filename, 'w') as f: f.write('{0} {1:.6f}'.format(p.pid, p.create_time()))
def _set_running(directory): """Write the current process information to disk. :param directory: The name of the directory where the process information will be written. """ filename = _get_pid_filename(directory) if isfile(str(filename)): raise PidFileExistsError p = Process() with open(filename, 'w') as f: f.write('{0} {1:.6f}'.format(p.pid, p.create_time()))
def test_running_valid_file(self): d = mkdtemp() f = realpath(join(d, '.pid')) try: _set_running(f) with open(f, 'r') as pid_file: process_info = pid_file.read().split() p = Process() self.assertEquals(p.pid, int(process_info[0])) self.assertEquals(p.create_time(), float(process_info[1])) finally: shutil.rmtree(d)
def test_running_valid_file(self): temp_d = mkdtemp() d = realpath(join(temp_d, '.pid')) mkdir(d) try: _set_running(d) with open(join(d, 'INFO'), 'r') as pid_file: process_info = pid_file.read().split() p = Process() self.assertEquals(p.pid, int(process_info[0])) self.assertEquals(p.create_time(), float(process_info[1])) finally: rmtree(temp_d)
def _is_running(pid_file): """ Determine whether or not the process is currently running. :param pid_file: The PID file containing the process information. :return: Whether or not the process is currently running. """ if not isfile(str(pid_file)): return False pid, create_time = _read_pid_file(pid_file) try: current = Process(pid) except NoSuchProcess: return False if current.create_time() == create_time: return True _delete(pid_file) return False
def process_with_stats(name, stdin, times=1): if times > 1: results = [] for _ in xrange(0, times): results.append(process_with_stats(name, stdin, 1)) result = {'execution': reduce(lambda a, c: a+c['execution']/float(times), results, 0), 'memory': reduce(lambda a, c: max(a, c['memory']), results, 0), 'output': results[0]['output'], 'status': results[0]['status']} return result process = Popen(name, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, close_fds=True) process.stdin.write(stdin.getvalue()) process.stdin.close() stats = Process(process.pid) memory_usage = stats.memory_info().rss while process.poll() is None: try: memory_usage = max(memory_usage, stats.memory_info().rss) except: memory_usage = 0 sleep(1/1000.0) execution_time = time() - stats.create_time() output = process.stdout.read() if memory_usage == 0 and process.returncode == 0: return process_with_stats(name, stdin) return {'execution': execution_time, 'memory': memory_usage, 'output': output, 'status': process.returncode}
def _is_running(directory): """ Determine whether or not the process is currently running. :param directory: The PID directory containing the process information. :return: True if there is another process running, False if there is not. """ if not isdir(str(directory)): return _is_locked(directory) try: pid, create_time = _read_pid_file(_get_pid_filename(directory)) except InvalidPidFileError: return _is_locked(directory, True) try: current = Process(pid) except NoSuchProcess: return _is_locked(directory, True) if current.create_time() != create_time: return _is_locked(directory, True) return True
def _on_server(self, args): os.chdir(self.original_dir) abs_args_path = os.path.abspath(args.path) component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read() out = { 'component_details': component_details, 'component_full_path': abs_args_path, 'component_host': current_host(), 'component_running': False, 'current_time': datetime.now().isoformat(), 'current_time_utc': datetime.utcnow().isoformat(), 'master_proc_connections': None, 'master_proc_pid': None, 'master_proc_name': None, 'master_proc_create_time': None, 'master_proc_create_time_utc': None, 'master_proc_username': None, 'master_proc_workers_no': None, 'master_proc_workers_pids': None, } master_proc_pid = None try: master_proc_pid = int(open(os.path.join(abs_args_path, MISC.PIDFILE)).read()) except(IOError, ValueError): # Ok, no such file or it's empty pass if master_proc_pid: out['component_running'] = True master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.children()) out['master_proc_connections'] = master_proc.connections() out['master_proc_pid'] = master_proc.pid out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time()).isoformat() out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time(), UTC).isoformat() out['master_proc_username'] = master_proc.username() out['master_proc_name'] = master_proc.name() out['master_proc_workers_no'] = len(workers_pids) out['master_proc_workers_pids'] = workers_pids for pid in workers_pids: worker = Process(pid) out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time()).isoformat() out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time(), UTC).isoformat() out['worker_{}_connections'.format(pid)] = worker.connections() if getattr(args, 'json', False): out['component_details'] = loads(out['component_details']) self.logger.info(dumps(out)) else: cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH cols_width = (elem.strip() for elem in cols_width.split(',')) cols_width = [int(elem) for elem in cols_width] table = Texttable() table.set_cols_width(cols_width) # Use text ('t') instead of auto so that boolean values don't get converted into ints table.set_cols_dtype(['t', 't']) rows = [['Key', 'Value']] rows.extend(sorted(out.items())) table.add_rows(rows) self.logger.info(table.draw())
def _get_create_time(process: psutil.Process): # create_time try: create_time = datetime.fromtimestamp(process.create_time()) except OSError: create_time = datetime.fromtimestamp(process.boot_time())
def setup_pyramid(comp, config): env = comp.env is_debug = env.core.debug # Session factory config.set_session_factory(WebSession) # Empty authorization policy. Why do we need this? # NOTE: Authentication policy is set up in then authentication component! authz_policy = ACLAuthorizationPolicy() config.set_authorization_policy(authz_policy) _setup_pyramid_debugtoolbar(comp, config) _setup_pyramid_tm(comp, config) _setup_pyramid_mako(comp, config) # COMMON REQUEST'S ATTRIBUTES config.add_request_method(lambda req: env, 'env', property=True) config.add_request_method( lambda req: req.path_info.lower().startswith('/api/'), 'is_api', property=True) # ERROR HANGLING comp.error_handlers = list() @comp.error_handlers.append def api_error_handler(request, err_info, exc, exc_info): if request.is_api or request.is_xhr: return exception.json_error_response(request, err_info, exc, exc_info, debug=is_debug) @comp.error_handlers.append def html_error_handler(request, err_info, exc, exc_info): return exception.html_error_response(request, err_info, exc, exc_info, debug=is_debug) def error_handler(request, err_info, exc, exc_info, **kwargs): for handler in comp.error_handlers: result = handler(request, err_info, exc, exc_info) if result is not None: return result config.registry.settings['error.err_response'] = error_handler config.registry.settings['error.exc_response'] = error_handler config.include(exception) config.add_tween( 'nextgisweb.pyramid.util.header_encoding_tween_factory', over=('nextgisweb.pyramid.exception.unhandled_exception_tween_factory', )) # INTERNATIONALIZATION # Substitute localizer from pyramid with our own, original is # too tied to translationstring, that works strangely with string # interpolation via % operator. def localizer(request): return request.env.core.localizer(request.locale_name) config.add_request_method(localizer, 'localizer', property=True) # Replace default locale negotiator with session-based one def locale_negotiator(request): return request.session.get('pyramid.locale', env.core.locale_default) config.set_locale_negotiator(locale_negotiator) # TODO: Need to get rid of translation dirs! # Currently used only to search for jed-files. from ..package import pkginfo as _pkginfo for pkg in _pkginfo.packages: dirname = resource_filename(pkg, 'locale') if os.path.isdir(dirname): config.add_translation_dirs(dirname) # STATIC FILES if is_debug: # In debug build static_key from proccess startup time rproc = Process(os.getpid()) # When running under control of uWSGI master process use master's startup time if rproc.name() == 'uwsgi' and rproc.parent().name() == 'uwsgi': rproc = rproc.parent() _logger.debug("Found uWSGI master process PID=%d", rproc.pid) # Use 4-byte hex representation of 1/5 second intervals comp.static_key = '/' + hex(int(rproc.create_time() * 5) % (2 ** 64)) \ .replace('0x', '').replace('L', '') _logger.debug("Using startup time static key [%s]", comp.static_key[1:]) else: # In production mode build static_key from pip freeze output comp.static_key = '/' + pip_freeze()[0] _logger.debug("Using pip freeze static key [%s]", comp.static_key[1:]) config.add_static_view('/static{}/asset'.format(comp.static_key), 'nextgisweb:static', cache_max_age=3600) config.add_route('amd_package', '/static{}/amd/*subpath'.format(comp.static_key)) \ .add_view(static_amd_file) # Collect external AMD-packages from other components amd_base = [] for c in comp._env.chain('amd_base'): amd_base.extend(c.amd_base) config.add_request_method(lambda r: amd_base, 'amd_base', property=True, reify=True) # RENDERERS config.add_renderer('json', json_renderer) # Filter for quick translation. Defines function tr, which we can use # instead of request.localizer.translate in mako templates. def tr_subscriber(event): event['tr'] = event['request'].localizer.translate config.add_subscriber(tr_subscriber, BeforeRender) # OTHERS config.add_route('home', '/').add_view(home) def ctpl(n): return 'nextgisweb:pyramid/template/%s.mako' % n config.add_route('pyramid.control_panel', '/control-panel', client=()) \ .add_view(control_panel, renderer=ctpl('control_panel')) config.add_route('pyramid.favicon', '/favicon.ico').add_view(favicon) config.add_route('pyramid.control_panel.pkginfo', '/control-panel/pkginfo').add_view( pkginfo, renderer=ctpl('pkginfo')) config.add_route('pyramid.control_panel.backup.browse', '/control-panel/backup/').add_view( backup_browse, renderer=ctpl('backup')) config.add_route( 'pyramid.control_panel.backup.download', '/control-panel/backup/{filename}').add_view(backup_download) config.add_route('pyramid.control_panel.cors', '/control-panel/cors').add_view(cors, renderer=ctpl('cors')) config.add_route('pyramid.control_panel.custom_css', '/control-panel/custom-css').add_view( custom_css, renderer=ctpl('custom_css')) config.add_route('pyramid.control_panel.logo', '/control-panel/logo').add_view(cp_logo, renderer=ctpl('logo')) config.add_route('pyramid.control_panel.system_name', '/control-panel/system-name').add_view( system_name, renderer=ctpl('system_name')) config.add_route('pyramid.control_panel.miscellaneous', '/control-panel/miscellaneous').add_view( miscellaneous, renderer=ctpl('miscellaneous')) config.add_route('pyramid.control_panel.home_path', '/control-panel/home_path').add_view( home_path, renderer=ctpl('home_path')) config.add_route('pyramid.locale', '/locale/{locale}').add_view(locale) comp.test_request_handler = None config.add_route('pyramid.test_request', '/test/request/') \ .add_view(test_request) config.add_route('pyramid.test_exception_handled', '/test/exception/handled') \ .add_view(test_exception_handled) config.add_route('pyramid.test_exception_unhandled', '/test/exception/unhandled') \ .add_view(test_exception_unhandled) config.add_route('pyramid.test_timeout', '/test/timeout') \ .add_view(test_timeout) comp.control_panel = dm.DynMenu( dm.Label('info', _("Info")), dm.Link( 'info/pkginfo', _("Package versions"), lambda args: (args.request.route_url('pyramid.control_panel.pkginfo'))), dm.Label('settings', _("Settings")), dm.Link( 'settings/core', _("Web GIS name"), lambda args: (args.request.route_url('pyramid.control_panel.system_name'))), dm.Link( 'settings/cors', _("Cross-origin resource sharing (CORS)"), lambda args: (args.request.route_url('pyramid.control_panel.cors'))), dm.Link( 'settings/custom_css', _("Custom CSS"), lambda args: (args.request.route_url('pyramid.control_panel.custom_css'))), dm.Link( 'settings/logo', _("Custom logo"), lambda args: (args.request.route_url('pyramid.control_panel.logo'))), dm.Link( 'settings/miscellaneous', _("Miscellaneous"), lambda args: (args.request.route_url('pyramid.control_panel.miscellaneous'))), dm.Link( 'settings/home_path', _("Home path"), lambda args: (args.request.route_url('pyramid.control_panel.home_path'))), ) if comp.options['backup.download']: comp.control_panel.add( dm.Link( 'info/backups', _("Backups"), lambda args: args.request. route_url('pyramid.control_panel.backup.browse')))