def get_cli(self, obj_type, parsed, argv):
"""get an object. see --help"""
try:
archive = parse_dates(parsed.get('--archive'))
except ValueError as err:
self.ABORT(str(err))
obj_name = parsed['name']
try:
obj = self.rcs.get(obj_type, obj_name, archive=archive)
if argv and argv[0]:
key = argv[0]
if key[:4] == 'obj.':
key = key[4:]
value = dictlib.dig_get(obj, key, '')
if parsed.get('--format') == 'txt':
if isinstance(value, str):
self.OUTPUT(value)
elif isinstance(value, list):
self.OUTPUT(", ".join(value))
else:
self.OUTPUT(json4human(value))
else:
self.OUTPUT(json4human(value))
else:
self.OUTPUT(json4human(obj))
except rfx.client.ClientError as err:
#self.ABORT("Exception: " + traceback.format_exc())
self.ABORT(str(err))
except NotFoundError:
self.ABORT("Cannot find object '" + obj_name + "'")
except Exception as err: # pylint: disable=broad-except
self.DEBUG("Exception: " + str(err))
self.ABORT("Exception: " + traceback.format_exc())
def do_match(match):
"""Internal function for processing a matched variable"""
match_key = match.group(1)
if match_key in os.environ:
return os.environ[match_key]
dict_match = dictlib.dig_get(dictionary, match_key, None)
if match_key != source and dict_match != None:
return str(dict_match)
self.NOTIFY("Unable to find expansion match for key '" +
match_key + "'")
return None
def slice_cli(self, obj_type, parsed, argv):
"""slice limit-expression extract-keys (comma delimited). see --help"""
name_rx = re.compile('^' + parsed['name-filter'])
limit = parsed['limit-expression']
extract = parsed['key']
if extract[:4] == "obj.":
extract = extract[4:]
extracted = set()
extracted_objs = dict()
for obj in Engine(base=self).session.list(obj_type, cols='*'):
if not name_rx.search(obj['name']):
continue
context = {'obj': dictlib.Obj(**obj), 'True': True, 'False': False}
try:
# pylint: disable=eval-used
if not eval(limit, {'__builtins__': {}, 'rx': re}, context):
continue
except KeyError:
print("key error")
continue
except: # pylint: disable=bare-except
traceback.print_exc()
value = dictlib.dig_get(obj, extract)
if not value:
continue
if isinstance(value, list):
extracted = extracted.union(value)
else:
extracted.add(value)
extracted_objs[obj['name']] = value
if parsed.get('--format', 'txt') == 'txt':
self.OUTPUT(" ".join(extracted) + "\n")
if parsed.get('--format', 'txt') == 'list':
# pylint: disable=consider-iterating-dictionary
for obj in sorted(extracted_objs.keys()):
self.OUTPUT("{}: {}".format(obj, extracted_objs[obj]))
else:
self.OUTPUT(ujson.dumps(extracted) + "\n")
def __init__(self, path=None, skeleton=False):
spec = dict()
if skeleton:
spec = {}
else:
if not path:
path = self._find_path(path)
if not path:
abort("Cannot find Polyform.yml")
with open(path) as infile:
try:
# CLoader faster, but is a difficult dependency
spec = yaml.full_load(infile)
except Exception as err: # pylint: disable=broad-except
abort(err)
# enforce structure around code testing
# TODO: this is only for python at the moment; split this out so
# pip always is run on GLOBAL_PIP, separate from whatever the language
# of the form wants
code_default = dict(type="codetest",
dependencies={"add": GLOBAL_PIP + ["pylint"]})
code_test = dictlib.dig_get(spec, "forms.codetest", None)
if code_test:
code_test = dictlib.union_setadd(code_test, code_default)
dictlib.dug(spec, "forms.codetest", code_test)
else:
dictlib.dug(spec, "forms.codetest", code_default)
# load up the polyform config
self.polyform = Polyform(keyword="polyform", config=self)
self.polyform._add_keys(spec)
# finish
self.polyform._finish()
def start(self, test=True):
"""
Startup script for webhook routing.
Called from agent start
"""
cherrypy.log = logger.CherryLog()
cherrypy.config.update({
'log.screen': False,
'log.access_file': '',
'log.error_file': ''
})
cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files)
logging.config.dictConfig({
'version': 1,
'formatters': {
'custom': {
'()': 'server.logger.Logger'
}
},
'handlers': {
'console': {
'level':'INFO',
'class': 'server.logger.Logger',
'formatter': 'custom',
'stream': 'ext://sys.stdout'
}
},
'loggers': {
'': {
'handlers': ['console'],
'level': 'INFO'
},
'cherrypy.access': {
'handlers': ['console'],
'level': 'INFO',
'propagate': False
},
'cherrypy.error': {
'handlers': ['console'],
'level': 'INFO',
'propagate': False
},
}
})
# lots of legacy stuff here which isn't used
defaults = {
'deploy_ver': 0, # usable for deployment tools
'server': {
'route_base': '/api/v1',
'port': 64000,
'host': '0.0.0.0'
},
'heartbeat': 10,
'status_report': 3600, # every hour
'requestid': True,
'refresh_maps': 300,
'cache': {
'housekeeper': 60,
'policies': 300,
'sessions': 300,
'groups': 300
},
'auth': {
'expires': 300
}
}
cfgin = None
# try docker secrets
if os.path.exists("/run/secrets/SERVER_CONFIG"):
with open("/run/secrets/SERVER_CONFIG") as infile:
cfgin = infile.read()
# try environ
if not cfgin:
cfgin = os.environ.get('SERVER_CONFIG')
if cfgin:
try:
cfgin = json2data(base64.b64decode(cfgin))
except: # pylint: disable=bare-except
try:
cfgin = json2data(cfgin)
except Exception as err: # pylint: disable=broad-except
traceback.print_exc()
logger.abort("Cannot process SERVER_CONFIG: " + str(err) + " from " + cfgin)
conf = Dict(dictlib.union(defaults, cfgin))
else:
logger.log("Unable to find configuration, using defaults!")
conf = Dict(defaults)
# cherry py global
cherry_conf = {
'server.socket_port': 64000,
'server.socket_host': '0.0.0.0'
}
if dictlib.dig_get(conf, 'server.port'): # .get('port'):
cherry_conf['server.socket_port'] = int(conf.server.port)
if dictlib.dig_get(conf, 'server.host'): # .get('host'):
cherry_conf['server.socket_host'] = conf.server.host
# if production mode
if test:
logger.log("Test mode enabled", type="notice")
conf['test_mode'] = True
else:
cherry_conf['environment'] = 'production'
conf['test_mode'] = False
sys.stdout.flush()
cherrypy.config.update(cherry_conf)
cherrypy.config.update({'engine.autoreload.on': False})
self.conf = conf
sys.path.append('.')
# # eventually
# for mod in self.endpoint_names:
# self.add_endpoint(mod)
# hack for now
# from . import polyform as polyform
from server.endpoints import polyform
self.add_endpoint('polyform', polyform)
# startup cleaning interval
def housekeeper(server):
for endpoint in server.endpoints:
try:
endpoint.handler.housekeeper(server)
except: # pylint: disable=bare-except
traceback.print_exc()
timeinterval.start(conf.auth.expires * 1000, housekeeper, self)
# mount routes
cherrypy.tree.mount(http.Health(server=self),
conf.server.route_base + "/health",
self.endpoint_conf)
int_mon = cherrypy.process.plugins.Monitor(cherrypy.engine,
self.monitor,
frequency=conf.heartbeat/2)
int_mon.start()
# whew, now start the server
logger.log("Base path={}".format(conf.server.route_base), type="notice")
cherrypy.engine.start()
cherrypy.engine.block()
def start(self, test=True):
"""
Startup script for webhook routing.
Called from agent start
"""
cherrypy.log = CherryLog()
cherrypy.config.update({
'log.screen': False,
'log.access_file': '',
'log.error_file': ''
})
cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files)
logging.config.dictConfig({
'version': 1,
'formatters': {
'custom': {
'()': 'rfxengine.server.cherry.Logger'
}
},
'handlers': {
'console': {
'level':'INFO',
'class':'rfxengine.server.cherry.Logger', #logging.StreamHandler',
'formatter': 'custom',
'stream': 'ext://sys.stdout'
}
},
'loggers': {
'': {
'handlers': ['console'],
'level': 'INFO'
},
'cherrypy.access': {
'handlers': ['console'],
'level': 'INFO',
'propagate': False
},
'cherrypy.error': {
'handlers': ['console'],
'level': 'INFO',
'propagate': False
},
}
})
defaults = {
'deploy_ver': 0, # usable for deployment tools
'server': {
'route_base': '/api/v1',
'port': 54000,
'host': '0.0.0.0'
},
'heartbeat': 10,
'status_report': 3600, # every hour
'requestid': False,
'refresh_maps': 300,
'cache': {
'housekeeper': 60,
'policies': 300,
'sessions': 300,
'groups': 300
},
'crypto': {
# pylint: disable=bad-continuation
# '000': {
# dd if=/dev...
# 'key': "",
# 'default': True,
# }
},
'db': {
'database': 'reflex_engine',
'user': '******'
},
'auth': {
'expires': 300
}
}
cfgin = None
# try docker secrets
if os.path.exists("/run/secrets/REFLEX_ENGINE_CONFIG"):
with open("/run/secrets/REFLEX_ENGINE_CONFIG") as infile:
cfgin = infile.read()
# try environ
if not cfgin:
cfgin = os.environ.get('REFLEX_ENGINE_CONFIG')
if cfgin:
try:
cfgin = json2data(base64.b64decode(cfgin))
except: # pylint: disable=bare-except
try:
cfgin = json2data(cfgin)
except Exception as err: # pylint: disable=broad-except
traceback.print_exc()
self.ABORT("Cannot process REFLEX_ENGINE_CONFIG: " +
str(err) + " from " + cfgin)
conf = dictlib.Obj(dictlib.union(defaults, cfgin))
else:
self.NOTIFY("Unable to find configuration, using defaults!")
conf = dictlib.Obj(defaults)
# cherry py global
cherry_conf = {
'server.socket_port': 9000,
'server.socket_host': '0.0.0.0'
}
if dictlib.dig_get(conf, 'server.port'): # .get('port'):
cherry_conf['server.socket_port'] = int(conf.server.port)
if dictlib.dig_get(conf, 'server.host'): # .get('host'):
cherry_conf['server.socket_host'] = conf.server.host
# if production mode
if test:
log("Test mode enabled", type="notice")
conf['test_mode'] = True
else:
cherry_conf['environment'] = 'production'
conf['test_mode'] = False
# db connection
self.dbm = mxsql.Master(config=conf.db, base=self, crypto=conf.get('crypto'))
# configure the cache
self.dbm.cache = rfxengine.memstate.Cache(**conf.cache.__export__())
self.dbm.cache.start_housekeeper(conf.cache.housekeeper)
# schema
schema = dbo.Schema(master=self.dbm)
schema.initialize(verbose=False, reset=False)
sys.stdout.flush()
cherrypy.config.update(cherry_conf)
endpoint_conf = {
'/': {
'response.headers.server': "stack",
'tools.secureheaders.on': True,
'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
'request.method_with_bodies': ('PUT', 'POST', 'PATCH'),
}
}
cherrypy.config.update({'engine.autoreload.on': False})
self.conf = conf
# startup cleaning interval
def clean_keys(dbm):
"""periodically called to purge expired auth keys from db"""
dbo.AuthSession(master=dbm).clean_keys()
timeinterval.start(conf.auth.expires * 1000, clean_keys, self.dbm)
# recheck policymaps every so often
def check_policymaps(dbm):
"""
periodically remap policy maps, incase somebody was
fidgeting where they shoudln't be
"""
dbo.Policyscope(master=dbm).remap_all()
timeinterval.start(conf.refresh_maps * 1000, check_policymaps, self.dbm)
# mount routes
cherrypy.tree.mount(endpoints.Health(conf, server=self),
conf.server.route_base + "/health",
endpoint_conf)
cherrypy.tree.mount(endpoints.Token(conf, server=self),
conf.server.route_base + "/token",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="config"),
conf.server.route_base + "/config",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="service"),
conf.server.route_base + "/service",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="pipeline"),
conf.server.route_base + "/pipeline",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="instance"),
conf.server.route_base + "/instance",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="build"),
conf.server.route_base + "/build",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="group"),
conf.server.route_base + "/group",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="apikey"),
conf.server.route_base + "/apikey",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="policy"),
conf.server.route_base + "/policy",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="policyscope"),
conf.server.route_base + "/policyscope",
endpoint_conf)
cherrypy.tree.mount(endpoints.Object(conf, server=self,
obj="state"),
conf.server.route_base + "/state",
endpoint_conf)
cherrypy.tree.mount(endpoints.InstancePing(conf, server=self),
conf.server.route_base + "/instance-ping",
endpoint_conf)
# cherrypy.tree.mount(endpoints.Compose(conf, server=self),
# conf.server.route_base + "/compose",
# endpoint_conf)
# setup our heartbeat monitor
int_mon = cherrypy.process.plugins.Monitor(cherrypy.engine,
self.monitor,
frequency=conf.heartbeat/2)
int_mon.start()
log("Base path={}".format(conf.server.route_base), type="notice")
cherrypy.engine.start()
cherrypy.engine.block()