def __init__(self, dm: DataModel, rule_lists: List[NacmRuleList]):
self.root = [] # type: List[RuleTreeNode]
for rl in rule_lists:
for rule in filter(lambda r: r.type == NacmRuleType.NACM_RULE_DATA,
rl.rules):
try:
ii = dm.parse_instance_id(rule.type_data.path)
except NonexistentSchemaNode as e:
error(epretty(e, __name__))
ii = []
nl = self.root
node_match_prev = None
for isel in ii:
node_match = (list(filter(lambda x: x.isel == isel, nl))
or [None])[0]
if node_match is None:
new_elem = RuleTreeNode()
new_elem.isel = isel
new_elem.up = node_match_prev
if isel is ii[-1]:
new_elem.rule = rule
nl.append(new_elem)
node_match_prev = new_elem
nl = new_elem.children
else:
if isel is ii[-1]:
node_match.rule = rule
node_match_prev = node_match
nl = node_match.children
def main(output_file, repository_path, tip_revision, tutorial_text_source,
output_format):
if repository_path is None:
raise click.UsageError(
"\nUnable to discover repository. Please specify one\n"
"either with the -r/--repository-path option or via\n"
"the GIT_DIR environment variable.")
# Convert string to enumerator:
tutorial_text_source = getattr(ProjectHistory.TutorialTextSource,
tutorial_text_source)
try:
if output_format == "bundle-zipfile":
compile_fun = compile_fromgitrepo
elif output_format == "html-only":
compile_fun = compile_html_only_fromgitrepo
else:
# (Shouldn't happen, because Click should enforce valid choice.)
raise click.UsageError(
f"unknown output_format \"{output_format}\"")
compile_fun(output_file, repository_path, tip_revision,
tutorial_text_source)
except TutorialStructureError as err:
colorlog.error(str(err))
return 1
else:
return 0
def login():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
try:
data = {
'id': ID,
'pw': PW,
}
r = requests.post('http://{}:{}/login'.format(HOST, PORT),
data=data,
allow_redirects=False)
if 'set-cookie' not in r.headers:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
cookie = r.headers['set-cookie']
cookie = [i for i in cookie.split(';') if 'connect.sid' in i][0]
headers = {'Cookie': cookie}
return headers
except:
colorlog.error('login failed')
os._exit(2)
return
def _put(ds: BaseDatastore, pth: str, username: str, data: str) -> HttpResponse:
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
rpc1 = RpcInfo()
rpc1.username = username
rpc1.path = url_path.rstrip("/")
try:
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse PUT data: " + epretty(e))
return HttpResponse.error(
HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e
)
try:
ds.lock_data(username)
try:
new_root = ds.update_node_rpc(ds.get_data_root_staging(rpc1.username), rpc1, json_data)
ds.add_to_journal_rpc(ChangeType.REPLACE, rpc1, json_data, new_root)
http_resp = HttpResponse.empty(HttpStatus.NoContent, status_in_body=False)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(
HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e
)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(
HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e
)
except NoHandlerError as e:
http_resp = HttpResponse.error(
HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e
)
except DataLockError as e:
http_resp = HttpResponse.error(
HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_LOCKDENIED,
exception=e
)
finally:
ds.unlock_data()
return http_resp
def check_none_gradients(grads, vars, ignore_none=False):
is_none = False
for grad, var in zip(grads, vars):
if grad is None:
colorlog.error(f"{var.name} has None gradient!")
is_none = True
if is_none and not ignore_none:
from IPython import embed
embed() # XXX DEBUG
def check_order():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
headers = login()
try:
# add to cart
r = requests.get(
'http://{}:{}/product/purchase?product-id=1&product-num=1'.format(
HOST, PORT),
headers=headers)
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
r = requests.get(
'http://{}:{}/product/add?product-id=2&product-num=2'.format(
HOST, PORT),
headers=headers)
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
r = requests.get('http://{}:{}/cart/purchase'.format(HOST, PORT),
headers=headers)
# check orders
r = requests.get('http://{}:{}/order'.format(HOST, PORT),
headers=headers)
soup = BeautifulSoup(r.text, 'html.parser')
orders = soup.find_all(class_='order-status')
# check number of orders
if len(orders) < 2:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
# check order status
for o in orders:
if o.string.strip() != 'pending':
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
except Exception as e:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
return
colorlog.info('"{}" passed'.format(current_function_name))
return
def jc_startup():
info("Backend: init")
# Create global API objects
so.KNOT = KnotConfig()
# Initialize Knot control interface
try:
so.KNOT.set_socket(config.CFG.root["KNOT"]["SOCKET"])
except KeyError:
error("Cannot find KNOT/SOCKET item in jetconf config file")
def check_auth():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
restricted_pages = ['order', 'cart', 'mypage']
for page in restricted_pages:
r = requests.get('http://{}:{}/order'.format(HOST, PORT))
if r.url != 'http://{}:{}/'.format(HOST, PORT):
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
colorlog.info('"{}" passed'.format(current_function_name))
def check_cart():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
try:
data = {
'id': ID,
'pw': PW,
}
r = requests.post('http://{}:{}/login'.format(HOST, PORT),
data=data,
allow_redirects=False)
if 'set-cookie' not in r.headers:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
cookie = r.headers['set-cookie']
cookie = [i for i in cookie.split(';') if 'connect.sid' in i][0]
headers = {
'Cookie': cookie,
}
r = requests.get(
'http://{}:{}/product/add?product-id=1&product-num=8'.format(
HOST, PORT),
headers=headers)
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
r = requests.get('http://{}:{}/cart'.format(HOST, PORT),
headers=headers)
ids = [
i.rstrip().lstrip() for i in r.text.split('\n') if 'cart-id' in i
]
for line in ids:
id_num = map(int, re.findall(r'([0-9]+)', line))
if len(id_num) != 1:
continue
r = requests.post('http://{}:{}/cart/remove'.format(HOST, PORT),
headers=headers,
data={'cart_id': id_num})
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
except Exception as e:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
return
colorlog.info('"{}" passed'.format(current_function_name))
return
def check_sendmsg():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
try:
data = {
'id': ID,
'pw': PW,
}
r = requests.post('http://{}:{}/login'.format(HOST, PORT),
data=data,
allow_redirects=False)
if 'set-cookie' not in r.headers:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
cookie = r.headers['set-cookie']
cookie = [i for i in cookie.split(';') if 'connect.sid' in i][0]
headers = {'Cookie': cookie}
msg = os.urandom(4).encode('base64').rstrip()
r = requests.post('http://{}:{}/mypage/sendMessage'.format(HOST, PORT),
headers=headers,
data={'msg': msg})
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
r = requests.get('http://{}:{}/mypage'.format(HOST, PORT),
headers=headers)
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
if msg not in r.text:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
except:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
return
colorlog.info('"{}" passed'.format(current_function_name))
return
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
self.client_cert = transport.get_extra_info("peercert")
ssl_context = transport.get_extra_info("ssl_object")
if ssl.HAS_ALPN:
agreed_protocol = ssl_context.selected_alpn_protocol()
else:
agreed_protocol = ssl_context.selected_npn_protocol()
if agreed_protocol is None:
error("Connection error, client does not support HTTP/2")
transport.close()
else:
self.conn.initiate_connection()
def check_connection():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
try:
r = requests.get('http://{}:{}'.format(HOST, PORT))
if r.status_code != 200:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
except:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
colorlog.info('"{}" passed'.format(current_function_name))
return
def generate_item(self, node_ii: InstanceRoute, username: str, staging: bool) -> JsonNodeT:
zone_obj = {}
# Request status of specific zone
KNOT.knot_connect()
domain_desired = node_ii[2].keys.get(("domain", None))
resp = KNOT.zone_status(domain_desired)
KNOT.knot_disconnect()
domain_name, status_data = tuple(resp.items())[0]
try:
zone_obj = {
"domain": domain_name.rstrip("."),
"class": "IN",
"serial": int(status_data.get("serial")),
"server-role": status_data.get("type")
}
except ValueError:
error("Error parsing Knot zone status data")
return zone_obj
def create_user_nacm(self, username: str):
# all_users = set()
# for gr in self.nacm_groups:
# for user in gr.users:
# all_users.add(user)
# for user in all_users:
# info("Creating personalized rule list for user \"{}\"".format(user))
# self._user_nacm_rpc[user] = UserNacm(self, user)
# if username not in all_users:
# raise NonexistentUserError
if not self.internal_data_lock.acquire(blocking=True, timeout=1):
error("Cannot acquire NACM config lock ")
return
info(
"Creating personalized rule list for user \"{}\"".format(username))
self._user_nacm_rpc[username] = UserRuleSet(self.dm, self, username)
self.internal_data_lock.release()
def _fill_state_roots(node: InstanceNode) -> InstanceNode:
if isinstance(node.value, ObjectValue):
if node.schema_node is state_root_sn.parent:
ii_gen = DataHelpers.node_get_ii(node)
sdh = STATE_DATA_HANDLES.get_handler(state_root_sch_pth)
if sdh is not None:
try:
if isinstance(sdh, ContainerNodeHandlerBase):
state_handler_val = sdh.generate_node(ii_gen, rpc.username, staging)
elif isinstance(sdh, ListNodeHandlerBase):
state_handler_val = sdh.generate_list(ii_gen, rpc.username, staging)
except Exception as e:
error("Error occured in state data generator (sn: {})".format(state_root_sch_pth))
error(epretty(e))
error("This state node will be omitted.")
else:
if state_root_sn.ns == state_root_sn.parent.ns:
nm_name = state_root_sn.qual_name[0]
else:
nm_name = state_root_sn.qual_name[1] + ":" + state_root_sn.qual_name[0]
# print("nm={}".format(nm_name))
node = node.put_member(nm_name, state_handler_val, raw=True).up()
else:
for key in node:
member = node[key]
node = _fill_state_roots(member).up()
elif isinstance(node.value, ArrayValue):
i = 0
arr_len = len(node.value)
while i < arr_len:
node = _fill_state_roots(node[i]).up()
i += 1
return node
def check_login():
frame = inspect.currentframe()
current_function_name = inspect.getframeinfo(frame).function
try:
# checks valid login process
data = {
'id': ID,
'pw': PW,
}
r = requests.post('http://{}:{}/login'.format(HOST, PORT),
data=data,
allow_redirects=False)
if 'set-cookie' not in r.headers:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
# checks invalid login process
data = {'id': 'asjdkfl;as', 'pw': 'qwueioprq'}
r = requests.post('http://{}:{}/login'.format(HOST, PORT),
data=data,
allow_redirects=False)
if 'set-cookie' in r.headers:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(1)
except:
colorlog.error('"{}" failed'.format(current_function_name))
os._exit(2)
colorlog.info('"{}" passed'.format(current_function_name))
return
def generate_list(self, node_ii: InstanceRoute, username: str, staging: bool) -> JsonNodeT:
zones_list = []
KNOT.knot_connect()
# Request status of all zones
resp = KNOT.zone_status()
KNOT.knot_disconnect()
for domain_name, status_data in resp.items():
try:
zone_obj = {
"domain": domain_name.rstrip("."),
"class": "IN",
"serial": int(status_data.get("serial")),
"server-role": status_data.get("type")
}
zones_list.append(zone_obj)
except ValueError:
error("Error parsing Knot zone status data")
return zones_list
def run(self):
FNULL = open(os.devnull, 'w')
for test_case in self.test_cases:
directory = os.path.dirname(test_case)
filename = os.path.basename(test_case)
filename_no_extension = os.path.splitext(filename)[0]
test_file_bc = filename_no_extension + ".bc"
print(" [*] Analyzing " + test_file_bc)
try:
p = subprocess.Popen([
"klee", "-max-time=" + str(self.timeout), "-output-dir=" +
self.output_dir + "/" + filename_no_extension, test_file_bc
],
stderr=FNULL,
stdout=FNULL)
print("the commandline is {}".format(p.args))
p.wait(timeout=self.timeout)
except CalledProcessError:
colorlog.error(
'One test failed during execution in Klee. Test named ' +
test["name"])
sys.stdout.write("\033[K")
sys.exit(-1)
except subprocess.TimeoutExpired:
p.kill()
if not os.path.isdir(self.output_dir + "/" +
filename_no_extension):
colorlog.error(
'One test failed during execution in Klee. Test named ' +
filename_no_extension)
break
FNULL.close()
def load(self):
super().load()
knot = KnotConfig()
# Initialize Knot control interface
try:
knot.set_socket(config.CFG.root["KNOT"]["SOCKET"])
except KeyError:
error("Cannot find KNOT/SOCKET item in jetconf config file")
# Read KnotDNS configuration and save it to the datastore
try:
knot.knot_connect()
knot_conf_json = knot.config_read()
knot.knot_disconnect()
new_root = self._data.put_member(
"dns-server:dns-server",
knot_conf_json["dns-server:dns-server"],
raw=True).top()
self.set_data_root(new_root)
except KnotError as e:
error("Cannot load KnotDNS configuration, reason: {}".format(
ErrorHelpers.epretty(e)))
def main():
config_file = "config.yaml"
# Check for Python version
if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher")
sys.exit(1)
# Get Jetconf version
jetconf_version = get_distribution("jetconf").version
# Parse command line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError:
print("Error: invalid argument detected.")
print_help()
sys.exit(1)
for opt, arg in opts:
if opt == "-c":
config_file = arg
elif opt == "-v":
print("Jetconf version {}".format(jetconf_version))
sys.exit(0)
elif opt == "-h":
print_help()
sys.exit(0)
# Load configuration
try:
load_config(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Validate configuration
try:
validate_config()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"])
# Go to background
pid = os.fork()
if pid != 0:
sys.exit(0)
os.setsid()
os.umask(0)
pid = os.fork()
if pid != 0:
sys.exit(0)
# Close standard file descriptors
os.close(sys.stdin.fileno())
os.close(sys.stdout.fileno())
os.close(sys.stderr.fileno())
fd_null = os.open("/dev/null", os.O_RDWR)
os.dup(fd_null)
os.dup(fd_null)
else:
# Setup color logging
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
},
secondary_log_colors={},
style='%')
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print version
info("Jetconf version {}".format(jetconf_version))
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Import backend modules
backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"]
try:
usr_state_data_handlers = import_module(backend_package +
".usr_state_data_handlers")
usr_conf_data_handlers = import_module(backend_package +
".usr_conf_data_handlers")
usr_op_handlers = import_module(backend_package + ".usr_op_handlers")
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError as e:
error(ErrorHelpers.epretty(e))
error("Cannot import backend package \"{}\". Exiting.".format(
backend_package))
sys.exit(1)
# Load data model
yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"]
yang_lib_str = resource_string(backend_package,
"yang-library-data.json").decode("utf-8")
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
# Datastore init
datastore = usr_datastore.UserDatastore(datamodel,
CONFIG_GLOBAL["DATA_JSON_FILE"],
with_nacm=CONFIG_NACM["ENABLED"])
try:
datastore.load()
except (FileNotFoundError, YangsonException) as e:
error("Could not load JSON datastore " +
CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Validate datastore on startup
try:
datastore.get_data_root().validate(ValidationScope.all,
ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register handlers for configuration data
usr_conf_data_handlers.register_conf_handlers(datastore)
# Register handlers for state data
usr_state_data_handlers.register_state_handlers(datastore)
# Register handlers for operations
op_internal.register_op_handlers(datastore)
usr_op_handlers.register_op_handlers(datastore)
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
def _post(self, pth: str, username: str, data: str) -> HttpResponse:
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
if len(url_split) > 1:
query_string = parse_qs(url_split[1])
else:
query_string = {}
rpc1 = RpcInfo()
rpc1.username = username
rpc1.path = url_path.rstrip("/")
rpc1.qs = query_string
# Skip NACM check for privileged users
if username in config.CFG.nacm["ALLOWED_USERS"]:
rpc1.skip_nacm_check = True
try:
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse POST data: " + epretty(e))
return HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
# Check if we are calling an action
ii = self.ds.parse_ii(rpc1.path, rpc1.path_format)
if isinstance(ii[-1], ActionName):
# Calling action on a node
ns = tuple(
filter(
lambda seg: hasattr(seg, "namespace") and
(seg.namespace is not None), ii))[-1].namespace
try:
input_args = json_data[ns + ":input"]
except KeyError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
else:
rpc1.op_input_args = input_args
try:
root_running = self.ds.get_data_root()
ret_data = self.ds.invoke_action_rpc(root_running, rpc1)
if ret_data is None:
http_resp = HttpResponse.empty(HttpStatus.NoContent,
status_in_body=False)
else:
if not isinstance(ret_data, str):
response = json.dumps(ret_data, indent=4)
else:
response = ret_data
http_resp = HttpResponse(HttpStatus.Ok,
response.encode(),
CTYPE_YANG_JSON)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except NoHandlerForOpError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e)
except (SchemaError, SemanticError) as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except (OpHandlerFailedError, StagingDataException,
YangsonException) as e:
http_resp = HttpResponse.error(
HttpStatus.InternalServerError,
RestconfErrType.Protocol,
ERRTAG_OPFAILED,
exception=e)
except ValueError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
else:
# Creating new node
try:
self.ds.lock_data(username)
try:
staging_root = self.ds.get_data_root_staging(rpc1.username)
new_root = self.ds.create_node_rpc(staging_root, rpc1,
json_data)
self.ds.add_to_journal_rpc(ChangeType.CREATE, rpc1,
json_data, *new_root)
http_resp = HttpResponse.empty(HttpStatus.Created)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except NoHandlerError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e)
except (InstanceValueError, YangsonException, ValueError) as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except InstanceAlreadyPresent as e:
http_resp = HttpResponse.error(HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_EXISTS,
exception=e)
except DataLockError as e:
http_resp = HttpResponse.error(HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_LOCKDENIED,
exception=e)
finally:
self.ds.unlock_data()
return http_resp
def _put(self, pth: str, username: str, data: str) -> HttpResponse:
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
rpc1 = RpcInfo()
rpc1.username = username
rpc1.path = url_path.rstrip("/")
# Skip NACM check for privileged users
if username in config.CFG.nacm["ALLOWED_USERS"]:
rpc1.skip_nacm_check = True
try:
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse PUT data: " + epretty(e))
return HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
try:
self.ds.lock_data(username)
try:
staging_root = self.ds.get_data_root_staging(rpc1.username)
new_root = self.ds.update_node_rpc(staging_root, rpc1,
json_data)
self.ds.add_to_journal_rpc(ChangeType.REPLACE, rpc1, json_data,
*new_root)
http_resp = HttpResponse.empty(HttpStatus.NoContent,
status_in_body=False)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except NoHandlerError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e)
except (InstanceValueError, StagingDataException, YangsonException,
ValueError) as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except DataLockError as e:
http_resp = HttpResponse.error(HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_LOCKDENIED,
exception=e)
finally:
self.ds.unlock_data()
return http_resp
def main(args):
# parse the args.
cli = ChemStorCLI(args)
z3 = Z3Solver(cli.config)
solution = z3.solve_constraints(z3.validate)
colorlog.error(solution)
def _post(ds: BaseDatastore, pth: str, username: str,
data: str) -> HttpResponse:
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
if len(url_split) > 1:
query_string = parse_qs(url_split[1])
else:
query_string = {}
rpc1 = RpcInfo()
rpc1.username = username
rpc1.path = url_path.rstrip("/")
rpc1.qs = query_string
# Skip NACM check for privileged users
if username in CONFIG_NACM["ALLOWED_USERS"]:
rpc1.skip_nacm_check = True
try:
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse POST data: " + epretty(e))
return HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
try:
ds.lock_data(username)
try:
try:
staging_root = ds.get_data_root_staging(rpc1.username)
except StagingDataException:
info("Starting transaction for user \"{}\"".format(
rpc1.username))
ds.make_user_journal(rpc1.username, None)
staging_root = ds.get_data_root_staging(rpc1.username)
new_root = ds.create_node_rpc(staging_root, rpc1, json_data)
ds.add_to_journal_rpc(ChangeType.CREATE, rpc1, json_data,
*new_root)
http_resp = HttpResponse.empty(HttpStatus.Created)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except NoHandlerError as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e)
except (InstanceValueError, YangsonException, ValueError) as e:
http_resp = HttpResponse.error(HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e)
except InstanceAlreadyPresent as e:
http_resp = HttpResponse.error(HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_EXISTS,
exception=e)
except DataLockError as e:
http_resp = HttpResponse.error(HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_LOCKDENIED,
exception=e)
finally:
ds.unlock_data()
return http_resp
def start(cfg: RosettaconfConfig, datamodel_name: str) -> None:
config.CFG = cfg
try:
cfg.validate()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO,
}.get(cast(str, cfg.glob["LOG_LEVEL"]), logging.INFO)
logging.root.handlers.clear()
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
"DEBUG": "cyan",
"INFO": "green",
"WARNING": "yellow",
"ERROR": "red",
"CRITICAL": "red",
},
secondary_log_colors={},
style="%",
)
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print version
info("NTC Rosetta Conf version {}".format("TBD"))
# Print configuration
cfg.print()
# Instantiate Jetconf main class
jc = Rosettaconf(cfg)
jetconf.JC = jc
try:
jc.init(get_data_model(datamodel_name))
except JetconfInitError as e:
error(str(e))
jc.cleanup()
# Exit
info("Exiting (error)")
sys.exit(1)
# Run Jetconf (this will block until shutdown)
jc.run()
jc.cleanup()
# Exit
info("Exiting")
sys.exit(0)
def main():
config_file = "config.yaml"
# Parse command line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "c:")
except getopt.GetoptError:
print("Invalid argument detected. Possible options are: -c (config file)")
sys.exit(1)
for opt, arg in opts:
if opt == "-c":
config_file = arg
# Load configuration
try:
load_config(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"]
)
# Go to background
pid = os.fork()
if pid != 0:
sys.exit(0)
os.setsid()
os.umask(0)
pid = os.fork()
if pid != 0:
sys.exit(0)
# Close standard file descriptors
os.close(sys.stdin.fileno())
os.close(sys.stdout.fileno())
os.close(sys.stderr.fileno())
fd_null = os.open("/dev/null", os.O_RDWR)
os.dup(fd_null)
os.dup(fd_null)
else:
# Setup color logging
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
},
secondary_log_colors={},
style='%'
)
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Load data model
yang_lib_file = os.path.join(CONFIG_GLOBAL["YANG_LIB_DIR"], "yang-library-data.json")
datamodel = DataHelpers.load_data_model(
CONFIG_GLOBAL["YANG_LIB_DIR"],
yang_lib_file
)
# Datastore init
datastore = JsonDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], "DNS data", with_nacm=False)
try:
datastore.load()
#datastore.load_yl_data(yang_lib_file)
except (FileNotFoundError, YangsonException) as e:
error("Could not load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
try:
datastore.get_data_root().validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register configuration data node listeners
# TODO
# Register op handlers
# TODO
# Create and register state data node listeners
usr_state_data_handlers.create_zone_state_handlers(STATE_DATA_HANDLES, datamodel)
# datastore callbacks TODO
datastore.commit_begin_callback = tsn_connect
datastore.commit_end_callback = tsn_disconnect
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
def commit(self, ds: "BaseDatastore") -> bool:
nacm_modified = False
if len(self._journal) == 0:
return False
if hash(ds.get_data_root()) == hash(self._root_origin):
info("Commiting new configuration (swapping roots)")
# Set new root
nr = self.get_root_head()
for change in self._journal:
nacm_modified = nacm_modified or change.nacm_modified
else:
info("Commiting new configuration (re-applying changes)")
nr = ds.get_data_root()
for change in self._journal:
nacm_modified = nacm_modified or change.nacm_modified
if change.change_type == ChangeType.CREATE:
nr = ds.create_node_rpc(nr, change.rpc_info,
change.input_data)[0]
elif change.change_type == ChangeType.REPLACE:
nr = ds.update_node_rpc(nr, change.rpc_info,
change.input_data)[0]
elif change.change_type == ChangeType.DELETE:
nr = ds.delete_node_rpc(nr, change.rpc_info)[0]
try:
# Validate syntax and semantics of new data
if CONFIG["GLOBAL"]["VALIDATE_TRANSACTIONS"] is True:
nr.validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Data validation error:")
error(epretty(e))
raise e
# Set new data root
ds.set_data_root(nr)
# Update NACM if NACM data has been affected by any edit
if nacm_modified and ds.nacm is not None:
ds.nacm.update()
# Call commit begin hook
begin_hook_failed = False
try:
ds.commit_begin_callback(self._transaction_opts)
except Exception as e:
error("Exception occured in commit_begin handler: {}".format(
epretty(e)))
begin_hook_failed = True
# Run schema node handlers
conf_handler_failed = False
if not begin_hook_failed:
try:
for change in self._journal:
ii = ds.parse_ii(change.rpc_info.path,
change.rpc_info.path_format)
ds.run_conf_edit_handler(ii, change)
except Exception as e:
error("Exception occured in edit handler: {}".format(
epretty(e)))
conf_handler_failed = True
# Call commit end hook
end_hook_failed = False
end_hook_abort_failed = False
if not (begin_hook_failed or conf_handler_failed):
try:
ds.commit_end_callback(self._transaction_opts, failed=False)
except Exception as e:
error("Exception occured in commit_end handler: {}".format(
epretty(e)))
end_hook_failed = True
if begin_hook_failed or conf_handler_failed or end_hook_failed:
try:
# Call commit_end callback again with "failed" argument set to True
ds.commit_end_callback(self._transaction_opts, failed=True)
except Exception as e:
error("Exception occured in commit_end handler (abort): {}".
format(epretty(e)))
end_hook_abort_failed = True
# Return to previous version of data and raise an exception if something went wrong
if begin_hook_failed or conf_handler_failed or end_hook_failed or end_hook_abort_failed:
ds.data_root_rollback(history_steps=1, store_current=False)
# Update NACM again after rollback
if nacm_modified and ds.nacm is not None:
ds.nacm.update()
raise ConfHandlerFailedError("(see logged)")
return True
def update(self):
lock_res = self.internal_data_lock.acquire(blocking=True, timeout=1)
if not lock_res:
error("NACM update: cannot acquire data lock")
return
self.nacm_groups = []
self.rule_lists = []
self._user_nacm_rpc = {}
try:
nacm_json = self.nacm_ds.get_data_root(
)["ietf-netconf-acm:nacm"].value
except NonexistentInstance:
raise ValueError(
"Data does not contain \"ietf-netconf-acm:nacm\" root element")
self.enabled = nacm_json["enable-nacm"]
if not self.enabled:
# NACM not enabled, no need to continue
self.internal_data_lock.release()
return
self.default_read = Action.PERMIT if nacm_json[
"read-default"] == "permit" else Action.DENY
self.default_write = Action.PERMIT if nacm_json[
"write-default"] == "permit" else Action.DENY
self.default_exec = Action.PERMIT if nacm_json[
"exec-default"] == "permit" else Action.DENY
for group in nacm_json["groups"]["group"]:
self.nacm_groups.append(
NacmGroup(group["name"], group["user-name"]))
for rule_list_json in nacm_json["rule-list"]:
rl = NacmRuleList()
rl.name = rule_list_json["name"]
rl.groups = rule_list_json["group"]
for rule_json in rule_list_json["rule"]:
rule = NacmRule()
rule.name = rule_json.get("name")
rule.comment = rule_json.get("comment")
rule.module = rule_json.get("module-name")
if rule_json.get("access-operations") is not None:
access_perm_list = rule_json["access-operations"]
if isinstance(access_perm_list, str) and (access_perm_list
== "*"):
rule.access = set(Permission)
elif isinstance(access_perm_list, collections.Iterable):
def perm_str2enum(perm_str: str):
return {
"read": Permission.NACM_ACCESS_READ,
"create": Permission.NACM_ACCESS_CREATE,
"update": Permission.NACM_ACCESS_UPDATE,
"delete": Permission.NACM_ACCESS_DELETE,
"exec": Permission.NACM_ACCESS_EXEC,
}.get(perm_str)
rule.access.update(map(perm_str2enum,
access_perm_list))
if rule_json.get("rpc-name") is not None:
if rule.type != NacmRuleType.NACM_RULE_NOTSET:
error(
"Invalid rule definition (multiple cases from rule-type choice): \"{}\""
.format(rule.name))
else:
rule.type = NacmRuleType.NACM_RULE_OPERATION
rule.type_data.rpc_names = rule_json.get(
"rpc-name").split()
if rule_json.get("notification-name") is not None:
if rule.type != NacmRuleType.NACM_RULE_NOTSET:
error(
"Invalid rule definition (multiple cases from rule-type choice): \"{}\""
.format(rule.name))
else:
rule.type = NacmRuleType.NACM_RULE_NOTIF
rule.type_data.ntf_names = rule_json.get(
"notification-name").split()
if rule_json.get("path") is not None:
if rule.type != NacmRuleType.NACM_RULE_NOTSET:
error(
"Invalid rule definition (multiple cases from rule-type choice): \"{}\""
.format(rule.name))
else:
rule.type = NacmRuleType.NACM_RULE_DATA
rule.type_data.path = rule_json["path"]
rule.action = Action.PERMIT if rule_json[
"action"] == "permit" else Action.DENY
rl.rules.append(rule)
self.rule_lists.append(rl)
self.internal_data_lock.release()
def _post(ds: BaseDatastore, pth: str, username: str, data: str) -> HttpResponse:
debug_httph("HTTP data received: " + data)
url_split = pth.split("?")
url_path = url_split[0]
if len(url_split) > 1:
query_string = parse_qs(url_split[1])
else:
query_string = {}
rpc1 = RpcInfo()
rpc1.username = username
rpc1.path = url_path.rstrip("/")
rpc1.qs = query_string
try:
json_data = json.loads(data) if len(data) > 0 else {}
except ValueError as e:
error("Failed to parse POST data: " + epretty(e))
return HttpResponse.error(
HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e
)
try:
ds.lock_data(username)
try:
new_root = ds.create_node_rpc(ds.get_data_root_staging(rpc1.username), rpc1, json_data)
ds.add_to_journal_rpc(ChangeType.CREATE, rpc1, json_data, new_root)
http_resp = HttpResponse.empty(HttpStatus.Created)
except NacmForbiddenError as e:
http_resp = HttpResponse.error(
HttpStatus.Forbidden,
RestconfErrType.Protocol,
ERRTAG_ACCDENIED,
exception=e
)
except (NonexistentSchemaNode, NonexistentInstance) as e:
http_resp = HttpResponse.error(
HttpStatus.NotFound,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e
)
except NoHandlerError as e:
http_resp = HttpResponse.error(
HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_OPNOTSUPPORTED,
exception=e
)
except (InstanceValueError, YangTypeError, ValueError) as e:
http_resp = HttpResponse.error(
HttpStatus.BadRequest,
RestconfErrType.Protocol,
ERRTAG_INVVALUE,
exception=e
)
except InstanceAlreadyPresent as e:
http_resp = HttpResponse.error(
HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_EXISTS,
exception=e
)
except DataLockError as e:
http_resp = HttpResponse.error(
HttpStatus.Conflict,
RestconfErrType.Protocol,
ERRTAG_LOCKDENIED,
exception=e
)
finally:
ds.unlock_data()
return http_resp
def main():
# Check for Python version
if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher")
sys.exit(1)
# Get Jetconf version
try:
jetconf_version = get_distribution("jetconf").version
except DistributionNotFound:
jetconf_version = "(not found)"
# Parse command line arguments
config_file = "config.yaml"
try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError:
print("Error: invalid argument detected.")
print_help()
sys.exit(1)
for opt, arg in opts:
if opt == "-c":
config_file = arg
elif opt == "-v":
print("Jetconf version {}".format(jetconf_version))
sys.exit(0)
elif opt == "-h":
print_help()
sys.exit(0)
# Load configuration
jc_config = config.JcConfig()
config.CFG = jc_config
try:
jc_config.load_file(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ValueError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Validate configuration
try:
jc_config.validate()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(jc_config.glob["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if jc_config.glob["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=jc_config.glob["LOGFILE"]
)
# Go to background
pid = os.fork()
if pid != 0:
sys.exit(0)
os.setsid()
os.umask(0)
pid = os.fork()
if pid != 0:
sys.exit(0)
# Close standard file descriptors
os.close(sys.stdin.fileno())
os.close(sys.stdout.fileno())
os.close(sys.stderr.fileno())
fd_null = os.open("/dev/null", os.O_RDWR)
os.dup(fd_null)
os.dup(fd_null)
else:
# Setup color logging
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
},
secondary_log_colors={},
style='%'
)
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print version
info("Jetconf version {}".format(jetconf_version))
# Print configuration
jc_config.print()
# Instantiate Jetconf main class
jc = jetconf.Jetconf(jc_config)
jetconf.JC = jc
try:
jc.init()
except JetconfInitError as e:
error(str(e))
jc.cleanup()
# Exit
info("Exiting (error)")
sys.exit(1)
# Run Jetconf (this will block until shutdown)
jc.run()
jc.cleanup()
# Exit
info("Exiting")
sys.exit(0)
