def get_cli(self, obj_type, parsed, argv): """get an object. see --help""" try: archive = parse_dates(parsed.get('--archive')) except ValueError as err: self.ABORT(str(err)) obj_name = parsed['name'] try: obj = self.rcs.get(obj_type, obj_name, archive=archive) if argv and argv[0]: key = argv[0] if key[:4] == 'obj.': key = key[4:] value = dictlib.dig_get(obj, key, '') if parsed.get('--format') == 'txt': if isinstance(value, str): self.OUTPUT(value) elif isinstance(value, list): self.OUTPUT(", ".join(value)) else: self.OUTPUT(json4human(value)) else: self.OUTPUT(json4human(value)) else: self.OUTPUT(json4human(obj)) except rfx.client.ClientError as err: #self.ABORT("Exception: " + traceback.format_exc()) self.ABORT(str(err)) except NotFoundError: self.ABORT("Cannot find object '" + obj_name + "'") except Exception as err: # pylint: disable=broad-except self.DEBUG("Exception: " + str(err)) self.ABORT("Exception: " + traceback.format_exc())
def do_match(match): """Internal function for processing a matched variable""" match_key = match.group(1) if match_key in os.environ: return os.environ[match_key] dict_match = dictlib.dig_get(dictionary, match_key, None) if match_key != source and dict_match != None: return str(dict_match) self.NOTIFY("Unable to find expansion match for key '" + match_key + "'") return None
def slice_cli(self, obj_type, parsed, argv): """slice limit-expression extract-keys (comma delimited). see --help""" name_rx = re.compile('^' + parsed['name-filter']) limit = parsed['limit-expression'] extract = parsed['key'] if extract[:4] == "obj.": extract = extract[4:] extracted = set() extracted_objs = dict() for obj in Engine(base=self).session.list(obj_type, cols='*'): if not name_rx.search(obj['name']): continue context = {'obj': dictlib.Obj(**obj), 'True': True, 'False': False} try: # pylint: disable=eval-used if not eval(limit, {'__builtins__': {}, 'rx': re}, context): continue except KeyError: print("key error") continue except: # pylint: disable=bare-except traceback.print_exc() value = dictlib.dig_get(obj, extract) if not value: continue if isinstance(value, list): extracted = extracted.union(value) else: extracted.add(value) extracted_objs[obj['name']] = value if parsed.get('--format', 'txt') == 'txt': self.OUTPUT(" ".join(extracted) + "\n") if parsed.get('--format', 'txt') == 'list': # pylint: disable=consider-iterating-dictionary for obj in sorted(extracted_objs.keys()): self.OUTPUT("{}: {}".format(obj, extracted_objs[obj])) else: self.OUTPUT(ujson.dumps(extracted) + "\n")
def __init__(self, path=None, skeleton=False): spec = dict() if skeleton: spec = {} else: if not path: path = self._find_path(path) if not path: abort("Cannot find Polyform.yml") with open(path) as infile: try: # CLoader faster, but is a difficult dependency spec = yaml.full_load(infile) except Exception as err: # pylint: disable=broad-except abort(err) # enforce structure around code testing # TODO: this is only for python at the moment; split this out so # pip always is run on GLOBAL_PIP, separate from whatever the language # of the form wants code_default = dict(type="codetest", dependencies={"add": GLOBAL_PIP + ["pylint"]}) code_test = dictlib.dig_get(spec, "forms.codetest", None) if code_test: code_test = dictlib.union_setadd(code_test, code_default) dictlib.dug(spec, "forms.codetest", code_test) else: dictlib.dug(spec, "forms.codetest", code_default) # load up the polyform config self.polyform = Polyform(keyword="polyform", config=self) self.polyform._add_keys(spec) # finish self.polyform._finish()
def start(self, test=True): """ Startup script for webhook routing. Called from agent start """ cherrypy.log = logger.CherryLog() cherrypy.config.update({ 'log.screen': False, 'log.access_file': '', 'log.error_file': '' }) cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files) logging.config.dictConfig({ 'version': 1, 'formatters': { 'custom': { '()': 'server.logger.Logger' } }, 'handlers': { 'console': { 'level':'INFO', 'class': 'server.logger.Logger', 'formatter': 'custom', 'stream': 'ext://sys.stdout' } }, 'loggers': { '': { 'handlers': ['console'], 'level': 'INFO' }, 'cherrypy.access': { 'handlers': ['console'], 'level': 'INFO', 'propagate': False }, 'cherrypy.error': { 'handlers': ['console'], 'level': 'INFO', 'propagate': False }, } }) # lots of legacy stuff here which isn't used defaults = { 'deploy_ver': 0, # usable for deployment tools 'server': { 'route_base': '/api/v1', 'port': 64000, 'host': '0.0.0.0' }, 'heartbeat': 10, 'status_report': 3600, # every hour 'requestid': True, 'refresh_maps': 300, 'cache': { 'housekeeper': 60, 'policies': 300, 'sessions': 300, 'groups': 300 }, 'auth': { 'expires': 300 } } cfgin = None # try docker secrets if os.path.exists("/run/secrets/SERVER_CONFIG"): with open("/run/secrets/SERVER_CONFIG") as infile: cfgin = infile.read() # try environ if not cfgin: cfgin = os.environ.get('SERVER_CONFIG') if cfgin: try: cfgin = json2data(base64.b64decode(cfgin)) except: # pylint: disable=bare-except try: cfgin = json2data(cfgin) except Exception as err: # pylint: disable=broad-except traceback.print_exc() logger.abort("Cannot process SERVER_CONFIG: " + str(err) + " from " + cfgin) conf = Dict(dictlib.union(defaults, cfgin)) else: logger.log("Unable to find configuration, using defaults!") conf = Dict(defaults) # cherry py global cherry_conf = { 'server.socket_port': 64000, 'server.socket_host': '0.0.0.0' } if dictlib.dig_get(conf, 'server.port'): # .get('port'): cherry_conf['server.socket_port'] = int(conf.server.port) if dictlib.dig_get(conf, 'server.host'): # .get('host'): cherry_conf['server.socket_host'] = conf.server.host # if production mode if test: logger.log("Test mode enabled", type="notice") conf['test_mode'] = True else: cherry_conf['environment'] = 'production' conf['test_mode'] = False sys.stdout.flush() cherrypy.config.update(cherry_conf) cherrypy.config.update({'engine.autoreload.on': False}) self.conf = conf sys.path.append('.') # # eventually # for mod in self.endpoint_names: # self.add_endpoint(mod) # hack for now # from . import polyform as polyform from server.endpoints import polyform self.add_endpoint('polyform', polyform) # startup cleaning interval def housekeeper(server): for endpoint in server.endpoints: try: endpoint.handler.housekeeper(server) except: # pylint: disable=bare-except traceback.print_exc() timeinterval.start(conf.auth.expires * 1000, housekeeper, self) # mount routes cherrypy.tree.mount(http.Health(server=self), conf.server.route_base + "/health", self.endpoint_conf) int_mon = cherrypy.process.plugins.Monitor(cherrypy.engine, self.monitor, frequency=conf.heartbeat/2) int_mon.start() # whew, now start the server logger.log("Base path={}".format(conf.server.route_base), type="notice") cherrypy.engine.start() cherrypy.engine.block()
def start(self, test=True): """ Startup script for webhook routing. Called from agent start """ cherrypy.log = CherryLog() cherrypy.config.update({ 'log.screen': False, 'log.access_file': '', 'log.error_file': '' }) cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files) logging.config.dictConfig({ 'version': 1, 'formatters': { 'custom': { '()': 'rfxengine.server.cherry.Logger' } }, 'handlers': { 'console': { 'level':'INFO', 'class':'rfxengine.server.cherry.Logger', #logging.StreamHandler', 'formatter': 'custom', 'stream': 'ext://sys.stdout' } }, 'loggers': { '': { 'handlers': ['console'], 'level': 'INFO' }, 'cherrypy.access': { 'handlers': ['console'], 'level': 'INFO', 'propagate': False }, 'cherrypy.error': { 'handlers': ['console'], 'level': 'INFO', 'propagate': False }, } }) defaults = { 'deploy_ver': 0, # usable for deployment tools 'server': { 'route_base': '/api/v1', 'port': 54000, 'host': '0.0.0.0' }, 'heartbeat': 10, 'status_report': 3600, # every hour 'requestid': False, 'refresh_maps': 300, 'cache': { 'housekeeper': 60, 'policies': 300, 'sessions': 300, 'groups': 300 }, 'crypto': { # pylint: disable=bad-continuation # '000': { # dd if=/dev... # 'key': "", # 'default': True, # } }, 'db': { 'database': 'reflex_engine', 'user': '******' }, 'auth': { 'expires': 300 } } cfgin = None # try docker secrets if os.path.exists("/run/secrets/REFLEX_ENGINE_CONFIG"): with open("/run/secrets/REFLEX_ENGINE_CONFIG") as infile: cfgin = infile.read() # try environ if not cfgin: cfgin = os.environ.get('REFLEX_ENGINE_CONFIG') if cfgin: try: cfgin = json2data(base64.b64decode(cfgin)) except: # pylint: disable=bare-except try: cfgin = json2data(cfgin) except Exception as err: # pylint: disable=broad-except traceback.print_exc() self.ABORT("Cannot process REFLEX_ENGINE_CONFIG: " + str(err) + " from " + cfgin) conf = dictlib.Obj(dictlib.union(defaults, cfgin)) else: self.NOTIFY("Unable to find configuration, using defaults!") conf = dictlib.Obj(defaults) # cherry py global cherry_conf = { 'server.socket_port': 9000, 'server.socket_host': '0.0.0.0' } if dictlib.dig_get(conf, 'server.port'): # .get('port'): cherry_conf['server.socket_port'] = int(conf.server.port) if dictlib.dig_get(conf, 'server.host'): # .get('host'): cherry_conf['server.socket_host'] = conf.server.host # if production mode if test: log("Test mode enabled", type="notice") conf['test_mode'] = True else: cherry_conf['environment'] = 'production' conf['test_mode'] = False # db connection self.dbm = mxsql.Master(config=conf.db, base=self, crypto=conf.get('crypto')) # configure the cache self.dbm.cache = rfxengine.memstate.Cache(**conf.cache.__export__()) self.dbm.cache.start_housekeeper(conf.cache.housekeeper) # schema schema = dbo.Schema(master=self.dbm) schema.initialize(verbose=False, reset=False) sys.stdout.flush() cherrypy.config.update(cherry_conf) endpoint_conf = { '/': { 'response.headers.server': "stack", 'tools.secureheaders.on': True, 'request.dispatch': cherrypy.dispatch.MethodDispatcher(), 'request.method_with_bodies': ('PUT', 'POST', 'PATCH'), } } cherrypy.config.update({'engine.autoreload.on': False}) self.conf = conf # startup cleaning interval def clean_keys(dbm): """periodically called to purge expired auth keys from db""" dbo.AuthSession(master=dbm).clean_keys() timeinterval.start(conf.auth.expires * 1000, clean_keys, self.dbm) # recheck policymaps every so often def check_policymaps(dbm): """ periodically remap policy maps, incase somebody was fidgeting where they shoudln't be """ dbo.Policyscope(master=dbm).remap_all() timeinterval.start(conf.refresh_maps * 1000, check_policymaps, self.dbm) # mount routes cherrypy.tree.mount(endpoints.Health(conf, server=self), conf.server.route_base + "/health", endpoint_conf) cherrypy.tree.mount(endpoints.Token(conf, server=self), conf.server.route_base + "/token", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="config"), conf.server.route_base + "/config", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="service"), conf.server.route_base + "/service", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="pipeline"), conf.server.route_base + "/pipeline", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="instance"), conf.server.route_base + "/instance", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="build"), conf.server.route_base + "/build", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="group"), conf.server.route_base + "/group", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="apikey"), conf.server.route_base + "/apikey", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="policy"), conf.server.route_base + "/policy", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="policyscope"), conf.server.route_base + "/policyscope", endpoint_conf) cherrypy.tree.mount(endpoints.Object(conf, server=self, obj="state"), conf.server.route_base + "/state", endpoint_conf) cherrypy.tree.mount(endpoints.InstancePing(conf, server=self), conf.server.route_base + "/instance-ping", endpoint_conf) # cherrypy.tree.mount(endpoints.Compose(conf, server=self), # conf.server.route_base + "/compose", # endpoint_conf) # setup our heartbeat monitor int_mon = cherrypy.process.plugins.Monitor(cherrypy.engine, self.monitor, frequency=conf.heartbeat/2) int_mon.start() log("Base path={}".format(conf.server.route_base), type="notice") cherrypy.engine.start() cherrypy.engine.block()