def configure_cron(self):
"""Configure cron settings."""
flog.debug("configuring cron")
env_vars = "PYTHONPATH={} FLAKE_SERVER={} FLAKE_TOOLS={} SHELL=/bin/bash".format(
os.environ.get("PYTHONPATH"),
os.environ.get("FLAKE_SERVER"),
os.environ.get("FLAKE_TOOLS"),
)
for cron_script, info in self.cron_config.items():
script = "python3 -u {} > {} 2>&1".format(info["script"],
info["log_file"])
cmd = "{vars} {script}".format(vars=env_vars, script=script)
# Restart script every 3 hours.
cron_cmd = '(crontab -l ; echo "{cron_time} {cmd}") | crontab -'.format(
cron_time="0 */3 * * *", cmd=cmd)
try:
# Initial start
flog.debug(cmd)
assert os.system(cmd) == 0, "Failed to start {}".format(
cron_script)
# Configure cron
flog.debug(cron_cmd)
assert os.system(cron_cmd) == 0, "Failed to configure cron"
except AssertionError as e:
flog.error(e)
return False
try:
assert os.system(
"/etc/init.d/cron restart") == 0, "Failed to start cron"
except AssertionError as e:
flog.error(e)
return False
return True
def add_files(self, force=False):
"""Add server files."""
try:
file_manager.configure_files(force)
return True
except Exception as ex:
flog.error(ex)
return False
def start(self):
"""Start iperf."""
try:
self.run_cmd()
return True
except AssertionError as e:
flog.error(e)
return False
def start(self):
"""Start pcap."""
try:
self.configure_pcap()
return True
except AssertionError as e:
flog.error(e)
return False
def apply_resume_after(lst, name):
idx = find_module_config(lst, name)
if idx == -1:
flog.error("Unknown module %s" % name)
return None
if idx == len(lst) - 1:
flog.error("No module after %s" % name)
return None
return lst[idx + 1:]
def get_data_handler(data_type, config, server_config):
"""Get data handler for data type."""
data_class = "%sFile" % data_type.capitalize()
try:
data_handler = eval(data_class)
except NameError as e:
flog.error("cannot find class %s, %s is not a valid data type." %
(data_class, data_type))
raise e
return data_handler(config, server_config)
def configure_server_rules(config_file=CONFIG,
with_filtering=True,
with_services=True):
"""Configure server rules."""
config = ConfigHandler(config_file)
try:
tc_manager = TrafficControl(config)
tc_manager.configure(with_filtering=with_filtering,
with_services=with_services)
tc_manager.show_rules()
return True
except AssertionError as ex:
flog.error(ex)
return False
def __init__(self, config):
"""Initialize http server."""
global DEFAULT_TIMEOUT
self.config = config
try:
self.port = int(self.config["dynamic_http"]["port"])
except (KeyError, ValueError) as e:
flog.error("Could not get port for dynamic http server.")
raise e
try:
DEFAULT_TIMEOUT = int(self.config["tcp_udp"]["timeout"])
except (KeyError, ValueError):
flog.warning(
"Could not use specified timeout, using default value: %d" %
DEFAULT_TIMEOUT)
self.address = ("0.0.0.0", self.port)
def run(self):
"""Run TCP/UDP client."""
flog.info(f"Connecting to {self.mode} server at {self.address}:{self.port}")
flog.debug(f"Echo: {self.echo}")
if not self.echo:
flog.debug(f"Message Size: {len(self.message)}")
if self.mode == "TCP":
self.run_tcp()
elif self.mode == "UDP":
self.run_udp()
else:
flog.error(
f"Cannot run client in mode: {self.mode}\nplease use either TCP or UDP"
)
return
flog.info(f"End {self.mode} Client")
def do_build(module_configs, log_dir, options):
result = BuildResult()
nb_modules = len(module_configs)
for idx, config in enumerate(module_configs):
name = config.flat_get("name")
assert name
flog.h1("%d/%d %s" % (idx + 1, nb_modules, name))
module = Module(config)
log_file_name = os.path.join(log_dir, name.replace("/", "_") + ".log")
log_file = open(log_file_name, "w")
runner = Runner(log_file, options.verbose)
# update/checkout
if options.switch_branch and module.has_checkout():
module.switch_branch(runner)
if not options.no_src:
try:
if module.has_checkout():
module.update(runner)
else:
module.checkout(runner)
except BatchBuildError, exc:
flog.error("%s failed to update/checkout: %s", name, exc)
flog.p("See %s", log_file_name)
result.vcs_fails.append([name, str(exc), log_file_name])
nanotify.notify(name, "Failed to update/checkout", icon="dialog-warning")
if options.fatal:
return result
# Build
try:
if not options.src_only:
if options.refresh_build:
module.refresh_build()
module.configure(runner)
module.build(runner)
module.install(runner)
nanotify.notify(name, "Build successfully", icon="dialog-ok")
except BatchBuildError, exc:
flog.error("%s failed to build: %s", name, exc)
flog.p("See %s", log_file_name)
result.build_fails.append([name, str(exc), log_file_name])
nanotify.notify(name, "Failed to build", icon="dialog-error")
if options.fatal:
return result
def start_cap():
parser = argparse.ArgumentParser(description="Pcap listener.")
parser.add_argument("--dev",
type=str,
help="Network device",
required=True)
parser.add_argument("--scheme",
type=str,
help="Scheme",
choices=["tcp", "udp"],
required=True)
parser.add_argument("--port", type=str, help="Port", required=True)
parser.add_argument("--log_dir",
type=str,
help="Log directory",
required=True)
parser.add_argument("--tmp_log",
type=str,
help="Temporary log file for listener",
required=True)
parser.add_argument("--timeout", type=int, help="Timeout", default=10)
parser.add_argument("--rotation_len",
type=int,
help="Number of logs to rotate",
default=10)
parser.add_argument(
"--headers_only",
type=str,
help="Only capture packet header",
choices=["True", "False"],
default="False",
)
args = parser.parse_args()
args.headers_only = eval(args.headers_only)
flog.debug(args)
pcap_listener = PcapListener(**vars(args))
pcap_listener.run()
flog.error("RUN ENDED")
def select_modules_from_config(config_name, module_names, base_dict):
def find_module(lst, name):
for dct in lst:
if dct["name"] == name:
return dct
return None
config = load_config_dict_by_name(config_name)
if not config:
flog.error("Could not find '%s' config file" % config_name)
return None
global_dict = config["global"]
module_dicts = config["modules"]
if not module_names:
return [CascadedConfig(x, global_dict, base_dict) for x in module_dicts]
lst = []
for module_name in module_names:
dct = find_module(module_dicts, module_name)
if dct is None:
flog.error("Unknown module %s" % module_name)
return None
lst.append(CascadedConfig(dct, global_dict, base_dict))
return lst
def main():
parser = OptionParser(usage=USAGE)
parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False,
help="Print command output to stdout")
parser.add_option("--dry-run",
action="store_true", dest="dry_run", default=False,
help="Just list what would be build")
parser.add_option("--no-src",
action="store_true", dest="no_src", default=False,
help="Do not update source code")
parser.add_option("--src-only",
action="store_true", dest="src_only", default=False,
help="Only update source code")
parser.add_option("--resume-from", dest="resume_from", default=None,
metavar="MODULE",
help="Resume build from MODULE")
parser.add_option("--resume-after", dest="resume_after", default=None,
metavar="MODULE",
help="Resume build after MODULE")
parser.add_option("--refresh-build",
action="store_true", dest="refresh_build", default=False,
help="Delete build dir")
parser.add_option("--switch-branch",
action="store_true", dest="switch_branch", default=False,
help="Switch to the branch defined for a module before updating")
parser.add_option("-l", "--list",
action="store_true", dest="list", default=False,
help="List available modules")
parser.add_option("--fatal",
action="store_true", dest="fatal", default=False,
help="Stop on first build failure")
(options, args) = parser.parse_args()
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%H:%M:%S", level=logging.DEBUG)
if options.list:
if len(args) > 0:
name = args[0]
config = load_config_dict_by_name(name)
if not config:
flog.error("No config named %s" % name)
return 1
print_project_modules(config)
else:
print_all_project_modules()
return 0
# Check devo name
devo_name = os.environ.get("DEVO_NAME", None)
if devo_name is None:
flog.error("No devo set up")
return 1
flog.p("Using devo '%s'", devo_name)
# Load config
if len(args) == 0:
parser.error("Missing args")
config_name = args[0]
if not config_name.endswith(".yaml"):
config_name += ".yaml"
module_names = args[1:]
base_dict = load_base_config_dict()
module_configs = select_modules_from_config(config_name, module_names, base_dict)
if module_configs is None:
return 1
if options.resume_from:
module_configs = apply_resume_from(module_configs, options.resume_from)
if options.resume_after:
module_configs = apply_resume_after(module_configs, options.resume_after)
if module_configs is None:
return 1
# Setup logging
log_dir = os.path.join(os.environ["DEVO_BUILD_BASE_DIR"], "log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if options.dry_run:
flog.p("Would build:")
for module_config in module_configs:
flog.li(module_config.flat_get("name"))
return 0
result = do_build(module_configs, log_dir, options)
flog.h1("Summary")
if result.vcs_fails:
fails = result.vcs_fails
flog.error("%d modules failed to update/checkout:", len(fails))
for name, msg, log_file_name in fails:
flog.li("%s: %s", name, msg)
flog.li("%s: see %s", name, log_file_name)
if result.build_fails:
fails = result.build_fails
flog.error("%d modules failed to build:", len(fails))
for name, msg, log_file_name in fails:
flog.li("%s: %s", name, msg)
flog.li("%s: see %s", name, log_file_name)
if result.vcs_fails or result.build_fails:
return 1
flog.p("All modules updated and built successfully")
return 0
def apply_resume_from(lst, name):
idx = find_module_config(lst, name)
if idx == -1:
flog.error("Unknown module %s" % name)
return None
return lst[idx:]