def main():
"""Set up and run the nowcast system message broker.
Set-up includes:
* Building the command-line parser, and parsing the command-line used
to launch the message broker
* Reading and parsing the configuration file given on the command-line
* Configuring the logging system as specified in the configuration file
* Log the message broker's PID, and the file path/name that was used to
configure it.
The set-up is repeated if the message broker process receives a HUP signal
so that the configuration can be re-loaded without having to stop and
re-start the message broker.
After the set-up is complete, launch the broker message queuing process.
See :command:`python -m nemo_nowcast.message_broker --help`
for details of the command-line interface.
"""
cli = CommandLineInterface(NAME, package="nemo_nowcast", description=__doc__)
cli.build_parser()
parsed_args = cli.parser.parse_args()
config = Config()
config.load(parsed_args.config_file)
msg = _configure_logging(config)
logger.info(f"running in process {os.getpid()}")
logger.info(f"read config from {config.file}")
logger.info(msg)
run(config)
def apply_config(self, new_config):
# check if new config changed
if(new_config != config.getDict()):
# disable while loading config
self.active = False
# TODO: deactivate queues
logging.info("Cleaning up actions and sensors")
self.cleanup_sensors()
self.cleanup_actions()
# TODO: check valid config file?!
# write config to file
try:
f = open('%s/worker/config.json'%(PROJECT_PATH),'w')
f.write(json.dumps(new_config))
f.close()
except Exception as e:
logging.exception("Wasn't able to write config file:\n%s" % e)
# set new config
config.load(PROJECT_PATH +"/worker/config.json")
if(config.get('active')):
logging.info("Activating actions and sensors")
self.setup_sensors()
self.setup_actions()
# TODO: activate queues
self.active = True
logging.info("Config saved successfully...")
else:
logging.info("Config didn't change")
def blu(rip: bool,
config_location: str,
log_config: str,
re_run: bool = False):
setup_logging(default_path=log_config)
config: Config = Config()
if config_location:
config.load(config_location)
else:
config.load('./blu.yml')
if rip:
if re_run:
makeMKV: MakeMKV = MakeMKV(MockCommander())
compressor: HandbrakeCLI = HandbrakeCLI(MockCommander())
else:
makeMKV: MakeMKV = MakeMKV(CommanderImpl())
compressor: HandbrakeCLI = HandbrakeCLI(CommanderImpl())
drives: List[Drive] = makeMKV.scan_drives()
identifier: Identifier = Identifier()
for drive in drives:
drive = makeMKV.scan_disc(drive)
identifier.identify(drive.disc)
if drive.disc.is_series():
ripSeries(compressor, config, drive, makeMKV, re_run)
else:
ripMovie(compressor, config, drive, makeMKV, re_run)
def apply_config(self, new_config):
# check if new config changed
if (new_config != config.getDict()):
# disable while loading config
self.active = False
# TODO: deactivate queues
logging.info("Cleaning up actions and sensors")
self.cleanup_sensors()
self.cleanup_actions()
# TODO: check valid config file?!
# write config to file
try:
f = open('%s/worker/config.json' % (PROJECT_PATH), 'w')
f.write(json.dumps(new_config))
f.close()
except Exception as e:
logging.exception("Wasn't able to write config file:\n%s" % e)
# set new config
config.load(PROJECT_PATH + "/worker/config.json")
if (config.get('active')):
logging.info("Activating actions and sensors")
self.setup_sensors()
self.setup_actions()
# TODO: activate queues
self.active = True
logging.info("Config saved successfully...")
else:
logging.info("Config didn't change")
def load_config(self, parser):
options, args = parser.parse_args(self._args)
self.config_path = options.config
def load_file(filename):
f = self._open(filename, 'r')
d = {'__file__': filename}
exec f.read() in d,d
return d
if options.config:
parser.set_defaults(**load_file(options.config))
elif len(args) == 0 or args[0] in ['start', 'restart', 'run']:
parser.error("a configuration file is required to start")
for ex in options.extensions:
try:
parser.set_defaults(**load_file(ex))
except IOError:
# couldn't open the file try to interpret as python
d = {}
exec ex in d,d
parser.set_defaults(**d)
# Now we parse args again with the config file settings as defaults
options, args = parser.parse_args(self._args)
config.load(options.__dict__)
return args
def load_config(self, parser):
options = parser.parse_args(self._args)
self.config_path = options.config
def load_file(filename):
f = self._open(filename, 'r')
d = {'__file__': filename}
exec f.read() in d, d
return d
parser.set_defaults(**load_file(options.config))
for ex in options.extensions:
try:
parser.set_defaults(**load_file(ex))
except IOError:
# couldn't open the file try to interpret as python
d = {}
exec ex in d, d
parser.set_defaults(**d)
# Now we parse args again with the config file settings as defaults
options = parser.parse_args(self._args)
config.load(options.__dict__)
self.action = options.action
self.normalize_config()
def load_config(self, parser):
options = parser.parse_args(self._args)
self.config_path = options.config
def load_file(filename):
f = self._open(filename, 'r')
d = {'__file__': filename}
exec f.read() in d,d
return d
parser.set_defaults(**load_file(options.config))
for ex in options.extensions:
try:
parser.set_defaults(**load_file(ex))
except IOError:
# couldn't open the file try to interpret as python
d = {}
exec ex in d,d
parser.set_defaults(**d)
# Now we parse args again with the config file settings as defaults
options = parser.parse_args(self._args)
config.load(options.__dict__)
self.action = options.action
self.normalize_config()
def run_server(config_file=""):
if config_file != "":
config_dir = os.path.dirname(config_file)
if (not os.path.exists(config_dir)) or (not os.path.isdir(config_dir)):
print "Error in the config file."
else:
config_dir = os.getcwd() + "/configs"
config_file = config_dir + "/genesis2.conf"
make_log(config_dir)
logger = logging.getLogger("genesis2")
logger.info("Genesis %s" % version())
if os.path.isfile(config_file):
logger.info("Using config file %s" % config_file)
else:
# Shutdown
logger.critical("The %s is not a file." % config_file)
exit(-1)
# Read config
config = Config()
if os.path.exists(config_file) and os.path.isfile(config_file):
config.load(config_file)
else:
logger.critical("The %s doesn't exist" % config_file)
exit(-1)
# (kudrom) TODO: I should delete the GenesisManager and substitute it with a Plugin
GenesisManager(config)
platform = detect_platform()
logger.info("Detected platform: %s" % platform)
# Load plugins
import genesis2.plugins
# Load apps
path_apps = config.get("genesis2", "path_apps", None)
if path_apps is None:
path_apps = os.getcwd() + "/apps"
logger.info("Using %s as path apps." % path_apps)
appmgr = AppManager(path_apps=path_apps)
appmgr.load_apps()
# (kudrom) TODO: Register a new ComponentMgr
# (kudrom) TODO: we should use an iptables plugin
# Make sure correct kernel modules are enabled
# genesis2.utils.shell('modprobe ip_tables')
if not hasattr(genesis2.apis, "PGenesis2Server"):
logger.error("There's no plugin for PGenesis2Server registered in the system")
exit(-1)
# The server is a plugin to ease its replacement
logger.info("Starting server")
server = getattr(genesis2.apis, "PGenesis2Server")
server.initialize(config)
server.serve_forever()
def __init__(self):
try: #TODO: this should be nicer...
logging.config.fileConfig(os.path.join(PROJECT_PATH, 'logging.conf'), defaults={'logfilename': 'manager.log'})
except Exception as e:
print("Error while trying to load config file for logging")
logging.info("Initializing manager")
try:
config.load(PROJECT_PATH +"/manager/config.json")
except ValueError: # Config file can't be loaded, e.g. no valid JSON
logging.exception("Wasn't able to load config file, exiting...")
quit()
try:
db.connect(PROJECT_PATH)
db.setup()
except:
logging.exception("Couldn't connect to database!")
quit()
self.notifiers = []
self.received_data_counter = 0
self.alarm_dir = "/var/tmp/secpi/alarms"
self.current_alarm_dir = "/var/tmp/secpi/alarms"
try:
self.data_timeout = int(config.get("data_timeout"))
except Exception: # if not specified in the config file we set a default value
self.data_timeout = 180
logging.debug("Couldn't find or use config parameter for data timeout in manager config file. Setting default value: %d" % self.data_timeout)
try:
self.holddown_timer = int(config.get("holddown_timer"))
except Exception: # if not specified in the config file we set a default value
self.holddown_timer = 210
logging.debug("Couldn't find or use config parameter for holddown timer in manager config file. Setting default value: %d" % self.holddown_timer)
self.holddown_state = False
self.num_of_workers = 0
self.connect()
# debug output, setups & state
setups = db.session.query(db.objects.Setup).all()
rebooted = False
for setup in setups:
logging.debug("name: %s active:%s" % (setup.name, setup.active_state))
if setup.active_state:
rebooted = True
if rebooted:
self.setup_notifiers()
self.num_of_workers = db.session.query(db.objects.Worker).join((db.objects.Action, db.objects.Worker.actions)).filter(db.objects.Worker.active_state == True).filter(db.objects.Action.active_state == True).count()
logging.info("Setup done!")
def __init__(self):
try: #TODO: this should be nicer...
logging.config.fileConfig(os.path.join(PROJECT_PATH, 'logging.conf'), defaults={'logfilename': 'manager.log'})
except Exception as e:
print("Error while trying to load config file for logging")
logging.info("Initializing manager")
try:
config.load(PROJECT_PATH +"/manager/config.json")
except ValueError: # Config file can't be loaded, e.g. no valid JSON
logging.exception("Wasn't able to load config file, exiting...")
quit()
try:
db.connect(PROJECT_PATH)
db.setup()
except:
logging.exception("Couldn't connect to database!")
quit()
self.notifiers = []
self.received_data_counter = 0
self.alarm_dir = "/var/tmp/secpi/alarms"
self.current_alarm_dir = "/var/tmp/secpi/alarms"
try:
self.data_timeout = int(config.get("data_timeout"))
self.holddown_timer = int(config.get("holddown_timer"))
except Exception: # if not specified in the config file we set default values for timeouts
logging.debug("Couldn't find config parameters for timeouts in config file, using default values for timeouts")
self.data_timeout = 10
self.holddown_timer = 30
self.holddown_state = False
self.num_of_workers = 0
self.connect()
# debug output, setups & state
setups = db.session.query(db.objects.Setup).all()
rebooted = False
for setup in setups:
logging.debug("name: %s active:%s" % (setup.name, setup.active_state))
if setup.active_state:
rebooted = True
if rebooted:
self.setup_notifiers()
self.num_of_workers = db.session.execute(text("select count(distinct w.id) as cnt from workers w join sensors s on w.id = s.worker_id join zones z on z.id = s.zone_id join zones_setups sz on sz.zone_id = z.id join setups se on se.id = sz.setup_id where se.active_state = 1 AND w.active_state = 1")).first()[0]
logging.info("Setup done!")
def main(argv):
logging.config.dictConfig(config.LOGGING)
conf = config.load(config.CONFIG_FILE)
checks = Checks.from_conf(conf)
# If CLI arg is provided, run the check.
if len(argv) >= 1 and argv[0] == "check":
project = None
name = None
if len(argv) > 1:
project = argv[1]
if len(argv) > 2:
name = argv[2]
try:
selected = checks.lookup(project=project, name=name)
except ValueError as e:
cprint(f"{e} in '{config.CONFIG_FILE}'", "red")
return 2
successes = []
for check in selected:
success = run_check(check)
successes.append(success)
return 0 if all(successes) else 1
# Otherwise, run the Web app.
app = init_app(checks)
web.run_app(app, host=config.HOST, port=config.PORT, print=False)
def load_config(self) -> dict:
"""
Load and parse config file
: return : dict
Loaded and parsed config
"""
return config.load(self.config_path)
def main():
logging.config.fileConfig('logging.conf')
parser = argparse.ArgumentParser(description='OSM Changeset diff filter')
parser.add_argument('-l', dest='log_level', default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='Set the log level')
parser.add_argument('--configdir', dest='configdir', default='.',
help='Set path to config file')
parser.add_argument('--db', dest='db_url', default='mongodb://localhost:27017/',
help='Set url for database')
subparsers = parser.add_subparsers()
parser_diff_fetch = subparsers.add_parser('diff-fetch')
parser_diff_fetch.set_defaults(func=diff_fetch)
parser_diff_fetch.add_argument('--initptr', action='store_true', default=False,
help='Reset OSM minutely diff pointer')
parser_diff_fetch.add_argument('-H', dest='history', help='Define how much history to fetch')
parser_diff_fetch.add_argument('--track', action='store_true', default=False,
help='Fetch current and future minutely diffs')
parser_diff_fetch.add_argument('--simulate', type=int, default=None, help='Simulate changeset observation')
parser_csets_filter = subparsers.add_parser('csets-filter')
parser_csets_filter.set_defaults(func=csets_filter)
parser_csets_analyze = subparsers.add_parser('csets-analyze')
parser_csets_analyze.set_defaults(func=csets_analyze)
parser_run_backends = subparsers.add_parser('run-backends')
parser_run_backends.set_defaults(func=run_backends)
parser_run_backends.add_argument('--track', action='store_true', default=False,
help='Track changes and re-run backends peridically')
parser_worker = subparsers.add_parser('worker')
parser_worker.set_defaults(func=worker)
parser_worker.add_argument('--track', action='store_true', default=False,
help='Track changes and re-run worker tasks peridically')
args = parser.parse_args()
logging.getLogger('').setLevel(getattr(logging, args.log_level))
config = configfile.Config()
config.load(path=args.configdir)
db = database.DataBase(url=args.db_url)
logger.debug('Connected to db: {}'.format(db))
return args.func(args, config, db)
def main(argv):
'''
Run the vmcache command line utility.
'''
try:
args = _parse(argv[1:])
if args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
else:
logging.basicConfig(level=logging.WARNING, stream=sys.stderr)
config = Config(args.configfile)
config.load()
_start_log(config, args)
args.func(config, args)
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
return 1
def main(argv):
'''
Run the vmcache command line utility.
'''
try:
args = _parse(argv[1:])
if args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
else:
logging.basicConfig(level=logging.WARNING, stream=sys.stderr)
config = Config(args.configfile)
config.load()
_start_log(config, args)
args.func(config, args)
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
return 1
def main():
args = _parse_args()
_setup_logging(args)
cfg = config.load(args.config)
db_path = cfg['anime'].getpath('database')
migrations.migrate(str(db_path))
cmd = AnimeCmd(cfg)
print(_INTRO)
cmd.cmdloop()
def initlog(path=None):
""" Set up logging """
logconf = config.load('logging.yaml', path)
logging.config.dictConfig(logconf)
log.critical("LOGLEVEL ENABLED: CRITICAL")
log.error("LOGLEVEL ENABLED: ERROR")
log.warn("LOGLEVEL ENABLED: WARN")
log.info("LOGLEVEL ENABLED: INFO")
log.debug("LOGLEVEL ENABLED: DEBUG")
def __init__(self):
self.actions = []
self.sensors = []
self.active = False
self.data_directory = "/var/tmp/secpi/worker_data"
self.zip_directory = "/var/tmp/secpi"
self.message_queue = [] # stores messages which couldn't be sent
try:
logging.config.fileConfig(os.path.join(PROJECT_PATH,
'logging.conf'),
defaults={'logfilename': 'worker.log'})
except Exception as e:
print("Error while trying to load config file for logging")
logging.info("Initializing worker")
try:
config.load(PROJECT_PATH + "/worker/config.json")
logging.debug("Config loaded")
except ValueError: # Config file can't be loaded, e.g. no valid JSON
logging.error("Wasn't able to load config file, exiting...")
quit()
time.sleep(60) # TEMPORARY FIX for #83
self.prepare_data_directory(self.data_directory)
self.connect()
# if we don't have a pi id we need to request the initial config, afterwards we have to reconnect
# to the queues which are specific to the pi id -> hence, call connect again
if not config.get('pi_id'):
logging.debug(
"No Pi ID found, will request initial configuration...")
self.fetch_init_config()
else:
logging.info("Setting up sensors and actions")
self.active = config.get('active')
self.setup_sensors()
self.setup_actions()
logging.info("Setup done!")
def __init__(self):
self.actions = []
self.sensors = []
self.active = False
self.data_directory = "/var/tmp/secpi/worker_data"
self.zip_directory = "/var/tmp/secpi"
self.message_queue = [] # stores messages which couldn't be sent
try:
logging.config.fileConfig(os.path.join(PROJECT_PATH, 'logging.conf'), defaults={'logfilename': 'worker.log'})
except Exception as e:
print("Error while trying to load config file for logging")
logging.info("Initializing worker")
try:
config.load(PROJECT_PATH +"/worker/config.json")
logging.debug("Config loaded")
except ValueError: # Config file can't be loaded, e.g. no valid JSON
logging.error("Wasn't able to load config file, exiting...")
quit()
time.sleep(60) # TEMPORARY FIX for #83
self.prepare_data_directory(self.data_directory)
self.connect()
# if we don't have a pi id we need to request the initial config, afterwards we have to reconnect
# to the queues which are specific to the pi id -> hence, call connect again
if not config.get('pi_id'):
logging.debug("No Pi ID found, will request initial configuration...")
self.fetch_init_config()
else:
logging.info("Setting up sensors and actions")
self.active = config.get('active')
self.setup_sensors()
self.setup_actions()
logging.info("Setup done!")
def arg_parser():
daemon = Daemon('watch_process.pid', main)
parser = argparse.ArgumentParser(
description='Stream download and upload, not only for bilibili.')
parser.add_argument('--version',
action='version',
version=f"v{__version__}")
parser.add_argument('-v',
'--verbose',
action="store_const",
const=logging.DEBUG,
help="Increase output verbosity")
parser.add_argument(
'--config',
type=argparse.FileType(encoding='UTF-8'),
help='Location of the configuration file (default "./config.yaml")')
subparsers = parser.add_subparsers(
help='Windows does not support this sub-command.')
# create the parser for the "start" command
parser_start = subparsers.add_parser('start',
help='Run as a daemon process.')
parser_start.set_defaults(func=daemon.start)
parser_stop = subparsers.add_parser(
'stop', help='Stop daemon according to "watch_process.pid".')
parser_stop.set_defaults(func=daemon.stop)
parser_restart = subparsers.add_parser('restart')
parser_restart.set_defaults(func=daemon.restart)
parser.set_defaults(func=lambda: asyncio.run(main()))
args = parser.parse_args()
config.load(args.config)
LOG_CONF.update(config.get('LOGGING', {}))
if args.verbose:
LOG_CONF['loggers']['biliup']['level'] = args.verbose
LOG_CONF['root']['level'] = args.verbose
logging.config.dictConfig(LOG_CONF)
if platform.system() == 'Windows':
return asyncio.run(main())
args.func()
def load_config(conf_dir):
pattern = os.path.join(conf_dir, "conf.d", "*.conf")
files = glob.glob(pattern)
# Trying to use invalid configuration directory is a user error or
# broken installation. Failing fast will help to fix the issue.
# https://github.com/oVirt/ovirt-imageio/issues/33
if not files:
raise ValueError(f"Could not find {pattern}")
# Vendor may override application defaults if needed.
pattern = os.path.join(VENDOR_CONF_DIR, "conf.d", "*.conf")
files.extend(glob.glob(pattern))
# Override files based on file name sort order:
# - /var/lib/ovirt-imageio/conf.d/75-vendor.conf overrides
# /etc/ovirt-imageio/conf.d/50-vdsm.conf.
# - /etc/ovirt-imageio/conf.d/99-user.conf overrides
# /var/lib/ovirt-imageio/conf.d/75-vendor.conf.
files.sort(key=os.path.basename)
return config.load(files)
def main():
args = parse_args()
configure_logger(args)
try:
log.info("Starting (pid=%s, version=%s)", os.getpid(), version.string)
cfg = config.load([os.path.join(args.conf_dir, "daemon.conf")])
server = Server(cfg)
signal.signal(signal.SIGINT, server.terminate)
signal.signal(signal.SIGTERM, server.terminate)
server.start()
try:
systemd.daemon.notify("READY=1")
log.info("Ready for requests")
while server.running:
signal.pause()
finally:
server.stop()
log.info("Stopped")
except Exception:
log.exception("Server failed")
sys.exit(1)
from sites.setups import SetupsPage
from sites.alarms import AlarmsPage
from sites.workers import WorkersPage
from sites.actions import ActionsPage
from sites.notifiers import NotifiersPage
from sites.actionparams import ActionParamsPage
from sites.notifierparams import NotifierParamsPage
from sites.sensorparams import SensorParamsPage
from sites.logs import LogEntriesPage
from sites.setupszones import SetupsZonesPage
from sites.workersactions import WorkersActionsPage
from sites.alarmdata import AlarmDataPage
config.load(PROJECT_PATH +"/webinterface/config.json")
class Root(object):
def __init__(self):
cherrypy.log("Initializing Webserver")
cherrypy.config.update({'request.error_response': self.handle_error})
cherrypy.config.update({'error_page.404': self.error_404})
cherrypy.config.update({'error_page.401': self.error_401})
self.sensors = SensorsPage()
self.zones = ZonesPage()
self.setups = SetupsPage()
self.alarms = AlarmsPage()
self.workers = WorkersPage()
def main(heartbeat_stop_callback=None):
global config
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False)
parser.add_option("-e", "--expected-hostname", dest="expected_hostname", action="store",
help="expected hostname of current host. If hostname differs, agent will fail", default=None)
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), logging_level)
global is_logger_setup
is_logger_setup = True
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
Logger.initialize_logger('resource_management', logging_level=logging_level)
# use the host's locale for numeric formatting
try:
locale.setlocale(locale.LC_ALL, '')
except locale.Error as ex:
logger.warning("Cannot set locale for ambari-agent. Please check your systemwide locale settings. Failed due to: {0}.".format(str(ex)))
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(config.get('agent','data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
#
# Iterate through the list of server hostnames and connect to the first active server
#
active_server = None
server_hostnames = hostname.server_hostnames(config)
connected = False
stopped = False
# Keep trying to connect to a server or bail out if ambari-agent was stopped
while not connected and not stopped:
for server_hostname in server_hostnames:
try:
server_ip = socket.gethostbyname(server_hostname)
server_url = config.get_api_url(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip)
except socket.error:
logger.warn("Unable to determine the IP address of the Ambari server '%s'", server_hostname)
# Wait until MAX_RETRIES to see if server is reachable
netutil = NetUtil(config, heartbeat_stop_callback)
(retries, connected, stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger)
# if connected, launch controller
if connected:
logger.info('Connected to Ambari server %s', server_hostname)
# Set the active server
active_server = server_hostname
# Launch Controller communication
controller = Controller(config, server_hostname, heartbeat_stop_callback)
controller.start()
while controller.is_alive():
time.sleep(0.1)
#
# If Ambari Agent connected to the server or
# Ambari Agent was stopped using stop event
# Clean up if not Windows OS
#
if connected or stopped:
ExitHelper().exit(0)
logger.info("finished")
break
pass # for server_hostname in server_hostnames
pass # while not (connected or stopped)
return active_server
def main():
logging.config.fileConfig('logging.conf')
parser = argparse.ArgumentParser(description='OSM Changeset diff filter')
parser.add_argument(
'-l',
dest='log_level',
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='Set the log level')
parser.add_argument('--configdir',
dest='configdir',
default='.',
help='Set path to config file')
parser.add_argument('--db',
dest='db_url',
default='mongodb://localhost:27017/',
help='Set url for database')
parser.add_argument('--metrics',
dest='metrics',
action='store_true',
default=False,
help='Enable metrics through Prometheus client API')
parser.add_argument('--metricsport',
dest='metricsport',
type=int,
default=8000,
help='Port through which to serve metrics')
subparsers = parser.add_subparsers()
parser_diff_fetch = subparsers.add_parser('diff-fetch')
parser_diff_fetch.set_defaults(func=diff_fetch)
parser_diff_fetch.add_argument('--initptr',
action='store_true',
default=False,
help='Reset OSM minutely diff pointer')
parser_diff_fetch.add_argument('-H',
dest='history',
help='Define how much history to fetch')
parser_diff_fetch.add_argument(
'--track',
action='store_true',
default=False,
help='Fetch current and future minutely diffs')
parser_diff_fetch.add_argument('--simulate',
type=int,
default=None,
help='Simulate changeset observation')
parser_csets_filter = subparsers.add_parser('csets-filter')
parser_csets_filter.set_defaults(func=csets_filter)
parser_csets_analyze = subparsers.add_parser('csets-analyze')
parser_csets_analyze.set_defaults(func=csets_analyze)
parser_run_backends = subparsers.add_parser('run-backends')
parser_run_backends.set_defaults(func=run_backends)
parser_run_backends.add_argument(
'--track',
action='store_true',
default=False,
help='Track changes and re-run backends periodically')
parser_worker = subparsers.add_parser('worker')
parser_worker.set_defaults(func=worker)
parser_worker.add_argument(
'--track',
action='store_true',
default=False,
help='Track changes and re-run worker tasks periodically')
parser_supervisor = subparsers.add_parser('supervisor')
parser_supervisor.set_defaults(func=supervisor)
parser_supervisor.add_argument(
'--track',
action='store_true',
default=False,
help='Track changes and re-run supervisor tasks periodically')
args = parser.parse_args()
logging.getLogger('').setLevel(getattr(logging, args.log_level))
config = configfile.Config()
config.load(path=args.configdir)
if args.metrics:
prometheus_client.start_http_server(args.metricsport)
if args.func == diff_fetch:
dbadm = True
else:
dbadm = False
db = database.DataBase(url=args.db_url, admin=dbadm)
logger.debug('Connected to db: {} (RW={})'.format(db, dbadm))
return args.func(args, config, db)
def __init__(self):
try: #TODO: this should be nicer...
logging.config.fileConfig(os.path.join(PROJECT_PATH,
'logging.conf'),
defaults={'logfilename': 'manager.log'})
except Exception as e:
print("Error while trying to load config file for logging")
logging.info("Initializing manager")
try:
config.load(PROJECT_PATH + "/manager/config.json")
except ValueError: # Config file can't be loaded, e.g. no valid JSON
logging.exception("Wasn't able to load config file, exiting...")
quit()
try:
db.connect(PROJECT_PATH)
db.setup()
except:
logging.exception("Couldn't connect to database!")
quit()
self.notifiers = []
self.received_data_counter = 0
self.alarm_dir = "/var/tmp/secpi/alarms"
self.current_alarm_dir = "/var/tmp/secpi/alarms"
try:
self.data_timeout = int(config.get("data_timeout"))
except Exception: # if not specified in the config file we set a default value
self.data_timeout = 180
logging.debug(
"Couldn't find or use config parameter for data timeout in manager config file. Setting default value: %d"
% self.data_timeout)
try:
self.holddown_timer = int(config.get("holddown_timer"))
except Exception: # if not specified in the config file we set a default value
self.holddown_timer = 210
logging.debug(
"Couldn't find or use config parameter for holddown timer in manager config file. Setting default value: %d"
% self.holddown_timer)
self.holddown_state = False
self.num_of_workers = 0
self.connect()
# debug output, setups & state
setups = db.session.query(db.objects.Setup).all()
rebooted = False
for setup in setups:
logging.debug("name: %s active:%s" %
(setup.name, setup.active_state))
if setup.active_state:
rebooted = True
if rebooted:
self.setup_notifiers()
self.num_of_workers = db.session.query(db.objects.Worker).join(
(db.objects.Action, db.objects.Worker.actions)).filter(
db.objects.Worker.active_state == True).filter(
db.objects.Action.active_state == True).count()
logging.info("Setup done!")
def load_config(args):
files = find_configs([VENDOR_CONF_DIR, args.conf_dir])
return config.load(files)
def main(heartbeat_stop_callback=None):
global config
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False)
parser.add_option("-e", "--expected-hostname", dest="expected_hostname", action="store",
help="expected hostname of current host. If hostname differs, agent will fail", default=None)
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), options.verbose)
global is_logger_setup
is_logger_setup = True
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), options.verbose)
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(config.get('agent','data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
server_hostname = hostname.server_hostname(config)
server_url = config.get_api_url()
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
try:
server_ip = socket.gethostbyname(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip)
except socket.error:
logger.warn("Unable to determine the IP address of the Ambari server '%s'", server_hostname)
# Wait until server is reachable
netutil = NetUtil(heartbeat_stop_callback)
retries, connected = netutil.try_to_connect(server_url, -1, logger)
# Ambari Agent was stopped using stop event
if connected:
# Launch Controller communication
controller = Controller(config, heartbeat_stop_callback)
controller.start()
controller.join()
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
ExitHelper.execute_cleanup()
stop_agent()
logger.info("finished")
def create_app(config_filename):
app = flask.Flask(__name__)
app.config.update(config.load(config_filename))
app.register_blueprint(handlers.root)
return app
def test_cbs(run_config_file_name: str, out_file_name: str = None, random_seed=None, permutations=None):
rc = config.load(run_config_file_name)
if permutations is not None:
rc.permutations = permutations
random.seed(random_seed)
if out_file_name:
base_out_path = pathlib.Path(out_file_name)
else:
base_out_path = pathlib.Path(rc.map_file_name).parent.joinpath(
f'routes-{time_utils.get_current_time_stamp()}.csv')
g = grid2d.Grid2D.from_file(pathlib.Path(rc.map_file_name))
if not rc.start:
rc.start = g.get_random_free_cell()
if not rc.end:
rc.end = g.get_random_free_cell({rc.start})
agent_count = len(rc.agents)
agents_have_start = False
agents_have_end = False
for a in rc.agents:
if a.get('start_cell'):
agents_have_start = True
if a.get('goal_cell'):
agents_have_end = True
if not agents_have_start:
start_cells = [rc.start] + g.find_free_cells_around(rc.start, agent_count - 1)
else:
start_cells = [a['start_cell'] for a in rc.agents]
if not agents_have_end:
end_cells = [rc.end] + g.find_free_cells_around(rc.end, agent_count - 1, set(start_cells))
else:
end_cells = [a['goal_cell'] for a in rc.agents]
for a, sc, gc in zip(rc.agents, start_cells, end_cells):
a['start_cell'] = sc
a['goal_cell'] = gc
LOG.info(f'STARTING mapf test, run_config: {rc}, base_out_path: {base_out_path}')
for permutation_idx in range(rc.permutations):
LOG.info(f'STARTED permutation {permutation_idx:03d}/{rc.permutations:03d}')
# random.shuffle(rc.agents)
agents = [agent.Agent(**a) for a in rc.agents]
cbs_finder = cbs.CbsMafpFinder(g)
agents_repo, total_cost = cbs_finder.find_path(agent_repository.AgentRepository(agents), astar.Searcher,
lambda agnt: dict(
h_func=mapf_heuristics.get_good_manhatten_like_heuristic(
agnt)))
for a in agents_repo.agents:
LOG.debug(
f"[{permutation_idx:03d}/{rc.permutations:3d}]:: Agent: {a.id}, path len: {len(a.path)} path cost: "
f"{a.path_cost}, expanded nodes: {a.expanded_nodes}")
print(g.to_str(a.cells_path(), a.start_cell, a.goal_cell, path_chr=str(a.id)[0]))
out_path = base_out_path.parent / (base_out_path.stem + f'-{permutation_idx:03d}' + base_out_path.suffix)
cbs_finder.save_paths(agents_repo, out_path)
LOG.info(f'FINISHED permutation {permutation_idx:03d}/{rc.permutations:03d} => {out_path}')
cbs_finder.validate_paths(g, agents_repo)
def e2e_parallel(run_config_file_name: str, out_dir: str = None, random_seed=None, permutations=None,
map_file_name: str = None, max_adv_agent_ds: int=None):# int = 2):
logging.getLogger('vgmapf.problems.mapf.multi_agent_pathfinding').setLevel(logging.INFO)
cores_count = multiprocessing.cpu_count()
LOG.info(f'Detected {cores_count} cores!')
print(run_config_file_name)
#rc1 = config.load('config-simple.yml')
rc1 = config.load(run_config_file_name)
if permutations is not None:
rc1.permutations = permutations
if map_file_name:
rc1.map_file_name = map_file_name
random.seed(random_seed)
max_adv_agent_ds = rc1.robust_route
if out_dir is None:
out_dir = pathlib.Path(__file__).parent / 'outputs' / time_utils.get_current_time_stamp()
else:
out_dir = pathlib.Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
out_dir1 = out_dir
grid = grid2d.Grid2D.from_file(pathlib.Path(rc1.map_file_name))
_update_start_and_goal_cells(rc1, grid)
print(grid.to_str(start=rc1.start, end=rc1.end))
start_time = timeit.default_timer()
max_agents = len(rc1.agents)
for swarm_ammount in range(2, max_agents+1): #(max_agents, max_agents+1):
rc = rc1
# Make folders for number of agents
swarm_out_dir = out_dir1 / str(len(rc.agents))
swarm_out_dir.mkdir(parents=True, exist_ok=True)
out_dir = swarm_out_dir
# Stage 1 - build normal paths
LOG.info('START 01 - building normal paths')
normal_paths_dir = out_dir / STAGE_01_NORMAL_PATHS
normal_paths_dir.mkdir(parents=True, exist_ok=True)
tasks = [(normal_paths_dir, grid, rc, permutation_idx) for permutation_idx in range(rc.permutations)]
with multiprocessing.Pool(processes=cores_count) as pool:
pool.starmap(_stage_1_normal_paths, tasks)
LOG.info('FINISH 01 - building normal paths')
LOG.info('STARTED 02 - run Robust Routes on normal paths')
# Stage 2 - robust routes
robust_paths_dir = out_dir / STAGE_02_ROBUST_PATHS
robust_paths_dir.mkdir(parents=True, exist_ok=True)
tasks = [(robust_paths_dir, grid, max_adv_agent_ds, p) for p in normal_paths_dir.iterdir()]
with multiprocessing.Pool(processes=cores_count) as pool:
pool.starmap(_stage_2_normal_robust, tasks)
LOG.info('FINISHED 02 - run Robust Routes on normal paths')
# #Stage 25 - run kamikaze on robust routes
# LOG.info('STARTED 025 - run kamikaze on robust routes')
#
# kamikaze_on_robust_paths_dir = out_dir / STAGE_025_KAMIKAZE
# kamikaze_on_robust_paths_dir.mkdir(parents=True, exist_ok=True)
#
# kamikaze_on_robust_results_summary = kamikaze_on_robust_paths_dir / '025-kamikaze_on_robust_paths.csv'
#
# # noinspection DuplicatedCode
# tasks = [(kamikaze_on_robust_paths_dir, rc, p) for p in robust_paths_dir.iterdir()]
# with multiprocessing.Pool(processes=cores_count) as pool:
# results = pool.starmap(_stage_25_kamikaze, tasks)
#
# if results:
# with kamikaze_on_robust_results_summary.open('w', newline='') as fresults:
# out_csv = csv.DictWriter(fresults, vars(results[0]).keys())
# out_csv.writeheader()
#
# for row in results:
# try:
# out_csv.writerow(vars(row))
# except Exception:
# LOG.warning(f'Failed writing row: {row}', exc_info=True)
#
# fresults.flush()
#
# LOG.info('FINISHED 025 - run kamikaze on robust routes')
# Stage 3 - run MDR on normal paths
LOG.info('STARTED 03 - run MDR on normal paths')
mdr_on_normal_paths_dir = out_dir / STAGE_03_MDR_NORMAL_PATHS
mdr_on_normal_paths_dir.mkdir(parents=True, exist_ok=True)
mdr_on_normal_results_summary = mdr_on_normal_paths_dir / '03-mdr_on_normal_paths-results.csv'
#tasks = [
# (mdr_on_normal_paths_dir, adv_agent.id, adv_agent_ds, rc, p) for p in normal_paths_dir.iterdir() for adv_agent
# in
# paths_serializer.load(p).agents for adv_agent_ds in range(1, max_adv_agent_ds + 1)
#]
tasks = [
(mdr_on_normal_paths_dir, adv_agent.id, adv_agent_ds, rc, p) for p in normal_paths_dir.iterdir() for
adv_agent
in
paths_serializer.load(p).agents for adv_agent_ds in range(1, max_adv_agent_ds + 1)
]
LOG.debug(f'stage_3 tasks:\n\t' + '\n\t'.join(str(x) for x in tasks))
with multiprocessing.Pool(processes=cores_count) as pool:
results = pool.starmap(_stage_3_normal_mdr, tasks)
if results:
with mdr_on_normal_results_summary.open('w', newline='') as fresults:
out_csv = csv.DictWriter(fresults, vars(results[0]).keys())
out_csv.writeheader()
for row in results:
try:
out_csv.writerow(vars(row))
except Exception:
LOG.warning( f'Failed writing row: {row}', exc_info=True)
LOG.info('FINISHED 03 - run MDR on normal paths')
LOG.info('STARTED 04 - run MDR on robust paths')
# Stage 4 - MDR on robust paths
mdr_on_robust_paths_dir = out_dir / STAGE_04_MDR_ROBUST_PATHS
mdr_on_robust_paths_dir.mkdir(parents=True, exist_ok=True)
mdr_on_robust_results_summary = mdr_on_robust_paths_dir / '04-mdr_on_robust_paths-results.csv'
tasks = [(mdr_on_robust_paths_dir, rc, p) for p in robust_paths_dir.iterdir()]
with multiprocessing.Pool(processes=cores_count) as pool:
results = pool.starmap(_stage_4_robust_mdr, tasks)
if results:
with mdr_on_robust_results_summary.open('w', newline='') as fresults:
out_csv = csv.DictWriter(fresults, vars(results[0]).keys())
out_csv.writeheader()
for row in results:
try:
out_csv.writerow(vars(row))
except Exception:
LOG.warning(f'Failed writing row: {row}', exc_info=True)
fresults.flush()
end_time = timeit.default_timer()
LOG.info(
f'FINISHED 04 - run MDR on robust paths, elapsed:{end_time - start_time:2f} = {datetime.timedelta(seconds=end_time - start_time)}')
del rc1.agents[-1]
def mapf(run_config_file_name: str, out_file_name: str = None, random_seed=None, permutations: int = None,
map_file_name: str = None):
rc = config.load(run_config_file_name)
if permutations is not None:
rc.permutations = permutations
if map_file_name:
rc.map_file_name = map_file_name
random.seed(random_seed)
if out_file_name:
base_out_path = pathlib.Path(out_file_name)
else:
timestamp = time_utils.get_current_time_stamp()
base_out_path = pathlib.Path(__file__).parent.joinpath(
'routes',
timestamp,
f'paths-{timestamp}.path'
)
base_out_path.parent.mkdir(parents=True, exist_ok=True)
g = grid2d.Grid2D.from_file(pathlib.Path(rc.map_file_name))
_update_start_and_goal_cells(rc, g)
LOG.info(f'STARTING mapf test, run_config: {rc}, base_out_path: {base_out_path}')
for permutation_idx in range(rc.permutations):
with benchmark_utils.time_it(f'Building path #{permutation_idx}'):
LOG.info(f'STARTED permutation {permutation_idx + 1:03d}/{rc.permutations:03d}')
if permutation_idx > 0:
random.shuffle(rc.agents)
agents = [agent.Agent(**a) for a in rc.agents]
mf = multi_agent_pathfinding.MapfFinder(g, agents)
mf.find_paths(astar.Searcher,
lambda agnt: dict(h_func=mapf_heuristics.get_good_manhatten_like_heuristic(agnt)))
for a in mf.agents:
LOG.debug(
f"[{permutation_idx + 1:03d}/{rc.permutations:03d}]:: Agent: {a.id}, path len: {len(a.path)} "
f"path cost: {a.path_cost}, expanded nodes: {a.expanded_nodes}")
print(g.to_str(a.cells_path(), a.start_cell, a.goal_cell, path_chr=str(a.id)[0]))
out_path_base = base_out_path.parent / (
base_out_path.stem + f'-{permutation_idx:03d}' + base_out_path.suffix)
mf.save_paths(out_path_base)
LOG.info(f'FINISHED permutation {permutation_idx + 1:03d}/{rc.permutations:03d} => {out_path_base}')
mf.validate_paths()
robust_route = RobustPathMode(rc.robust_route)
if robust_route == RobustPathMode.OFFLINE:
makespan_original = mf.agents_repo.get_makespan()
for agnt in agents:
if not agnt.is_adversarial:
agnt.path = None
agnt.path_cost = 0
agnt.expanded_nodes = 0
mf_robust = multi_agent_pathfinding.MapfFinder(g, agents,
adv_agent_radiuses={a.id: a.damage_steps * 2 for a in
agents
if a.is_adversarial})
mf_robust.find_paths(astar.Searcher,
lambda agnt: dict(h_func=mapf_heuristics.get_good_manhatten_like_heuristic(agnt)))
for a in mf_robust.agents:
LOG.debug(
f"[{permutation_idx + 1:03d}/{rc.permutations:03d}]:: Agent: {a.id}, path len: {len(a.path)} "
f"path cost: {a.path_cost}, expanded nodes: {a.expanded_nodes}")
print(g.to_str(a.cells_path(), a.start_cell, a.goal_cell, path_chr=str(a.id)[0]))
out_path_robust = base_out_path.parent / (base_out_path.stem + f'-{permutation_idx:03d}-robust'
+ base_out_path.suffix)
mf_robust.save_paths(out_path_robust)
LOG.info(f'FINISHED permutation {permutation_idx + 1:03d}/{rc.permutations:03d} => {out_path_robust}')
mf_robust.validate_paths()
makespan_robust = mf_robust.agents_repo.get_makespan()
LOG.info(f'The difference in makespan is {makespan_robust - makespan_original}')
return base_out_path.parent
import att_iot_gateway.att_iot_gateway as IOT #provide cloud support
def sigterm_handler(_signo, _stack_frame):
# Raises SystemExit(0):
sys.exit(0)
def on_connected():
if _connectedEvent: # if we reconnect during runtime, there is no more connected event.
_connectedEvent.set()
_connectedEvent = Event()
try:
signal.signal(signal.SIGTERM, sigterm_handler)
config.load()
IOT.on_connected = on_connected # so we can wait for starting to run the modules untill we are connected with the broker.
cloud.connect(modules.Actuate, processors.onAssetValueChanged)
processors.load(config.processors)
modules.load(config.modules)
modules.syncGateway()
logging.info("waiting for mqtt connection before starting all plugins")
_connectedEvent.wait()
_connectedEvent = None # when we are done we no longer need this event, so remove.Only needed to wait, so plugins can send init values
modules.run()
if config.configs.has_option('webServer', 'enabled') and config.configs.get('webServer', 'enabled') == True: # only load webserver if activated. Not all plugins need this, not all gateways want have a webserver running ex: fifthplay
import pygate_core.webServer as webServer
webServer.run()
while 1:
time.sleep(3)
except (KeyboardInterrupt, SystemExit):
def __init__(self, parent):
super(MainPanel, self).__init__(parent)
self.parent = parent
self.games = config.get_games()
self.game = None
self.game_watcher = None
while not self.games:
error = wx.MessageDialog(parent=self,
message="You have no games profiles set up. Replacing config with default.",
caption="Info", style=wx.OK | wx.ICON_INFORMATION)
error.ShowModal()
error.Destroy()
config.new()
config.load()
self.games = config.get_games()
self.profile = wx.ComboBox(parent=self, choices=[game.name for game in self.games],
style=wx.CB_READONLY)
self.profile.SetSelection(0)
self.track_list = ObjectListView(parent=self, style=wx.LC_REPORT | wx.BORDER_SUNKEN, sortable=True,
useAlternateBackColors=False)
self.track_list.SetEmptyListMsg("You currently do not have any sound files for this game.")
self.track_list.SetColumns([
ColumnDefn(title="#", fixedWidth=50, valueGetter="index", stringConverter="%i"),
ColumnDefn(title="Title", width=250, valueGetter="name", minimumWidth=150, isSpaceFilling=True),
ColumnDefn(title="Aliases", width=300, valueGetter="get_aliases", minimumWidth=200, isSpaceFilling=True),
ColumnDefn(title="Bind", width=75, valueGetter="bind", minimumWidth=50, maximumWidth=120)
])
self.track_list.rowFormatter = lambda x, y: x.SetTextColour(wx.RED) if y.get_aliases() == NO_ALIASES else None
self.selected_track = None
self.game_select(event=None)
refresh_button = wx.Button(parent=self, label="Refresh tracks")
self.start_stop_button = wx.Button(parent=self, label="Start")
convert_button = wx.Button(parent=self, label="Audio converter")
download_button = wx.Button(parent=self, label="Audio downloader")
top_sizer = wx.BoxSizer(wx.VERTICAL) # Root sizer
profile_sizer = wx.BoxSizer(wx.VERTICAL) # For the profile selection
olv_sizer = wx.BoxSizer(wx.VERTICAL) # For the ObjectListView
button_sizer = wx.BoxSizer(wx.HORIZONTAL) # Start/Stop and Refresh buttons
profile_sizer.Add(self.profile, 0, wx.LEFT | wx.RIGHT | wx.EXPAND | wx.ALIGN_TOP, 5)
olv_sizer.Add(self.track_list, 1, wx.LEFT | wx.RIGHT | wx.EXPAND | wx.ALIGN_TOP, 5)
button_sizer.Add(self.start_stop_button, 0, wx.ALL | wx.ALIGN_LEFT, 5)
button_sizer.Add(refresh_button, 0, wx.ALL | wx.ALIGN_LEFT, 5)
button_sizer.Add(convert_button, 0, wx.ALL | wx.ALIGN_LEFT, 5)
button_sizer.Add(download_button, 0, wx.ALL | wx.ALIGN_LEFT, 5)
top_sizer.Add(profile_sizer, 0, wx.ALL | wx.EXPAND, 5)
top_sizer.Add(olv_sizer, 1, wx.ALL | wx.EXPAND, 5)
top_sizer.Add(button_sizer, 0, wx.ALL | wx.EXPAND, 5)
top_sizer.SetSizeHints(self.parent)
self.SetSizerAndFit(top_sizer)
# Context menu
self.context_menu = wx.Menu()
set_aliases = self.context_menu.Append(wx.ID_ANY, "Set custom aliases")
clear_aliases = self.context_menu.Append(wx.ID_ANY, "Clear custom aliases")
set_bind = self.context_menu.Append(wx.ID_ANY, "Set bind")
clear_bind = self.context_menu.Append(wx.ID_ANY, "Clear bind")
clear_all = self.context_menu.Append(wx.ID_CLEAR, "Clear EVERYTHING (all tracks)")
trim_file = self.context_menu.Append(wx.ID_CUT, "Trim audio file")
self.Bind(wx.EVT_LIST_ITEM_RIGHT_CLICK, handler=self.list_right_click, source=self.track_list)
self.Bind(wx.EVT_MENU, handler=self.set_aliases, source=set_aliases)
self.Bind(wx.EVT_MENU, handler=self.clear_aliases, source=clear_aliases)
self.Bind(wx.EVT_MENU, handler=self.set_bind, source=set_bind)
self.Bind(wx.EVT_MENU, handler=self.clear_bind, source=clear_bind)
self.Bind(wx.EVT_MENU, handler=self.clear_all, source=clear_all)
self.Bind(wx.EVT_MENU, handler=self.trim_file, source=trim_file)
self.Bind(wx.EVT_COMBOBOX, handler=self.game_select, source=self.profile)
self.Bind(wx.EVT_BUTTON, handler=self.refresh, source=refresh_button)
self.Bind(wx.EVT_BUTTON, handler=self.start_stop, source=self.start_stop_button)
self.Bind(wx.EVT_BUTTON, handler=self.convert, source=convert_button)
self.Bind(wx.EVT_BUTTON, handler=self.download, source=download_button)
# self.Bind(wx.EVT_SIZE, handler=self.on_size)
self.Bind(wx.EVT_CLOSE, handler=self.on_exit)
import ast
from utils.config_manager import ConfigManager
APP_NAME = "notifyme"
APP_CHARSET = 'UTF-8'
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Config file paths
CONFIGURATION_JSON_FILE = './config.json'
CONFIGURATION_JOSN_FILE_DEV = './config_development.json'
config = ConfigManager(CONFIGURATION_JSON_FILE, CONFIGURATION_JOSN_FILE_DEV)
LOG_ROOT_PATH = config.load('LOG_ROOT_PATH', 'logging', 'root_path')
RABBITMQ_SERVER = config.load('RABBITMQ_SERVER', 'bus', 'host')
RABBITMQ_USER = config.load('RABBITMQ_USER', 'bus', 'user')
RABBITMQ_PASSWORD = config.load('RABBITMQ_PASSWORD', 'bus', 'password')
RABBITMQ_QUEUE = config.load('RABBITMQ_QUEUE', 'bus', 'queue_name')
RABBIRMQ_EXCHANGE_ERROR = config.load('RABBIRMQ_EXCHANGE_ERROR', 'bus',
'error_exchange')
SMTP_EMAIL = config.load('SMTP_EMAIL', 'smtp', 'email')
SMTP_HOST = config.load('SMTP_HOST', 'smtp', 'server')
SMTP_PORT = config.load('SMTP_PORT', 'smtp', 'port')
SMTP_FROM_NAME = config.load('SMTP_FROM_NAME', 'smtp', 'name')
SMTP_PASS = config.load('SMTP_PASS', 'smtp', 'password')
SEND_EMAILS = config.load('SMTP_SEND', 'smtp', 'name')
SMTP_TTLS = config.load('SMTP_TTLS', 'smtp', 'ttls')
def _init_config(self):
cfg = config.load(self._path)
if not self._cfg:
self._cfg = cfg
return
self._override_cfg(self._cfg.getNode(), cfg.getNode())
from sites.zones import ZonesPage
from sites.setups import SetupsPage
from sites.alarms import AlarmsPage
from sites.workers import WorkersPage
from sites.actions import ActionsPage
from sites.notifiers import NotifiersPage
from sites.actionparams import ActionParamsPage
from sites.notifierparams import NotifierParamsPage
from sites.sensorparams import SensorParamsPage
from sites.logs import LogEntriesPage
from sites.setupszones import SetupsZonesPage
from sites.workersactions import WorkersActionsPage
from sites.alarmdata import AlarmDataPage
config.load(PROJECT_PATH + "/webinterface/config.json")
class Root(object):
def __init__(self):
cherrypy.log("Initializing Webserver")
cherrypy.config.update({'request.error_response': self.handle_error})
cherrypy.config.update({'error_page.404': self.error_404})
cherrypy.config.update({'error_page.401': self.error_401})
self.sensors = SensorsPage()
self.zones = ZonesPage()
self.setups = SetupsPage()
self.alarms = AlarmsPage()
self.workers = WorkersPage()
def main(*config_files,
args=None,
config_dirs=(),
commands=(),
config_dict=None):
cwd = os.getcwd()
if cwd not in sys.path:
sys.path.insert(0, cwd)
context.config.search_dirs.extend(config_dirs)
if not commands:
p = Path(sys.argv[0])
if __package__ in (p.parent.name, p.name):
commands += __name__,
elif p.name.startswith('__'):
commands += p.parent.name,
else:
commands += p.name,
plugins = search_plugins()
plugins.extend(search_plugins(*commands, force=True))
for i in plugins:
i.add_arguments(parser)
if args is None:
args, argv = parser.parse_known_args()
cmds = list(commands)
while argv and not argv[0].startswith('-'):
cmds.append(argv.pop(0))
if getattr(args, 'config', None):
config_files += tuple(args.config)
if getattr(args, 'config_stdin', None):
assert not args.interact, 'Can not be used --config-stdin' \
' with --interact'
config_dict = utils.load_from_fd(sys.stdin.buffer)
if args.logging:
logging.basicConfig(level=args.logging.upper())
else:
cmds, argv = list(commands), []
config = context.config
plugins.extend(search_plugins(*cmds))
for p in plugins:
args, argv = p.parse_known_args(args=argv, namespace=args)
config.load(*p.configs)
config.update(p.get_config())
cmds = [cmd for cmd in cmds if cmd not in sys.modules]
config.load(*config_files)
config_dict and config.update(config_dict)
def sum_g(list_groups):
if list_groups:
return set(reduce(operator.add, list_groups))
run = partial(loop_run,
group_resolver=GroupResolver(
include=sum_g(args.groups),
exclude=sum_g(args.exclude_groups),
all_groups=args.exclude_groups is not None,
default=True,
),
cmds=cmds,
argv=argv,
ns=args,
prompt=PROMPT)
try:
if args.multiprocessing:
with context.processes():
print(PROMPT)
logger = multiprocessing.get_logger()
processes = process_iter(config.get('processes', {}))
for p in processes:
logger.info('Create process %s', p['name'])
p['process'] = create_process(p)
while True:
multiprocessing.connection.wait(
map(lambda x: x['process'].sentinel, processes), )
for p in processes:
proc = p['process'] # type: multiprocessing.Process
if not proc.is_alive():
logger.critical('Recreate process %s', p['name'])
p['process'] = create_process(p)
time.sleep(1)
elif args.interact:
from .core.interact import shell
args.print = lambda *args: None
shell(run)
elif args.interact_kernel:
from .core.interact import kernel
kernel(run)
else:
run()
except KeyboardInterrupt:
pass
finally:
for p in multiprocessing.active_children():
os.kill(p.pid, signal.SIGTERM)
t = time.monotonic()
sentinels = [p.sentinel for p in multiprocessing.active_children()]
while sentinels and time.monotonic() - t < args.shutdown_timeout:
multiprocessing.connection.wait(sentinels)
sentinels = [p.sentinel for p in multiprocessing.active_children()]
while multiprocessing.active_children():
print('killall children')
for p in multiprocessing.active_children():
os.kill(p.pid, signal.SIGKILL)
time.sleep(0.3)
def main(heartbeat_stop_callback=None):
global config
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False)
parser.add_option("-e", "--expected-hostname", dest="expected_hostname", action="store",
help="expected hostname of current host. If hostname differs, agent will fail", default=None)
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
current_user = getpass.getuser()
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(), logging_level)
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
Logger.initialize_logger('resource_management', logging_level=logging_level)
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
bind_signal_handlers(agentPid)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(config.get('agent','data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
server_hostname = hostname.server_hostname(config)
server_url = config.get_api_url()
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
try:
server_ip = socket.gethostbyname(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)', server_url, server_ip)
except socket.error:
logger.warn("Unable to determine the IP address of the Ambari server '%s'", server_hostname)
# Wait until server is reachable
netutil = NetUtil(heartbeat_stop_callback)
retries, connected = netutil.try_to_connect(server_url, -1, logger)
# Ambari Agent was stopped using stop event
if connected:
# Launch Controller communication
controller = Controller(config, heartbeat_stop_callback)
controller.start()
controller.join()
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
ExitHelper.execute_cleanup()
stop_agent()
logger.info("finished")
from contextlib import closing
import logging.config
from alembic import context
from sqlalchemy import pool
from great import config
from great.models.core import METADATA
from great.web import engine_from_config
CONFIG = config.load()
CONTEXT_CONFIG = dict(
sqlalchemy_module_prefix="sqlalchemy.",
target_metadata=METADATA,
)
logging.config.fileConfig(context.config.config_file_name)
def run_migrations_offline(config):
"""
Run migrations in 'offline' mode.
This configures the context with just a URL and not an Engine,
though an Engine is acceptable here as well. By skipping the Engine
creation we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the script
output.
"""
async def sappservice(config_filename, loop):
config = Config(config_filename, "", "")
config.load()
logging.config.dictConfig(copy.deepcopy(config["logging"]))
log: logging.Logger = logging.getLogger("sappservice")
log.info("Initializing matrix spring lobby appservice")
log.info(f"Config file: {config_filename}")
# def exception_hook(etype, value, trace):
# log.debug(traceback.format_exception(etype, value, trace))
#
# sys.excepthook = exception_hook
################
#
# Initialization
#
################
mebibyte = 1024**2
server = config["homeserver.address"]
domain = config["homeserver.domain"]
verify_ssl = config["homeserver.verify_ssl"]
as_token = config["appservice.as_token"]
hs_token = config["appservice.hs_token"]
bot_localpart = config["appservice.bot_username"]
max_body_size = config["appservice.max_body_size"]
hostname = config["appservice.hostname"]
port = config["appservice.port"]
client_name = config["spring.client_name"]
rooms = config["bridge.rooms"]
upgrade_table = UpgradeTable()
db = PostgresDatabase(config["appservice.database"], upgrade_table)
await db.start()
state_store_db = PgASStateStore(db=db)
await state_store_db.upgrade_table.upgrade(db.pool)
appserv = AppService(
server=server,
domain=domain,
verify_ssl=verify_ssl,
as_token=as_token,
hs_token=hs_token,
bot_localpart=bot_localpart,
loop=loop,
id='appservice',
state_store=state_store_db,
aiohttp_params={"client_max_size": max_body_size * mebibyte})
spring_lobby_client = SpringLobbyClient(appserv, config, loop=loop)
await db.start()
await appserv.start(hostname, port)
await spring_lobby_client.start()
################
#
# Lobby events
#
################
@spring_lobby_client.bot.on("tasserver")
async def on_lobby_tasserver(message):
log.debug(f"on_lobby_tasserver {message}")
if message.client.name == client_name:
message.client._login()
@spring_lobby_client.bot.on("clients")
async def on_lobby_clients(message):
log.debug(f"on_lobby_clients {message}")
if message.client.name != client_name:
channel = message.params[0]
clients = message.params[1:]
await spring_lobby_client.join_matrix_room(channel, clients)
@spring_lobby_client.bot.on("joined")
async def on_lobby_joined(message, user, channel):
log.debug(f"LOBBY JOINED user: {user.username} room: {channel}")
if user.username != "appservice":
await spring_lobby_client.join_matrix_room(channel,
[user.username])
@spring_lobby_client.bot.on("left")
async def on_lobby_left(message, user, channel):
log.debug(f"LOBBY LEFT user: {user.username} room: {channel}")
if channel.startswith("__battle__"):
return
if user.username == "appservice":
return
await spring_lobby_client.leave_matrix_room(channel, [user.username])
@spring_lobby_client.bot.on("said")
async def on_lobby_said(message, user, target, text):
if message.client.name == client_name:
await spring_lobby_client.said(user, target, text)
@spring_lobby_client.bot.on("saidex")
async def on_lobby_saidex(message, user, target, text):
if message.client.name == client_name:
await spring_lobby_client.saidex(user, target, text)
# @spring_lobby_client.bot.on("denied")
# async def on_lobby_denied(message):
# return
# # if message.client.name != client_name:
# # user = message.client.name
# # await spring_appservice.register(user)
# @spring_lobby_client.bot.on("adduser")
# async def on_lobby_adduser(message):
# if message.client.name != client_name:
# username = message.params[0]
#
# if username == "ChanServ":
# return
# if username == "appservice":
# return
#
# await spring_lobby_client.login_matrix_account(username)
# @spring_lobby_client.bot.on("removeuser")
# async def on_lobby_removeuser(message):
# if message.client.name != client_name:
# username = message.params[0]
#
# if username == "ChanServ":
# return
# if username == "appservice":
# return
#
# await spring_lobby_client.logout_matrix_account(username)
@spring_lobby_client.bot.on("accepted")
async def on_lobby_accepted(message):
log.debug(f"message Accepted {message}")
await spring_lobby_client.config_rooms()
await spring_lobby_client.sync_matrix_users()
@spring_lobby_client.bot.on("failed")
async def on_lobby_failed(message):
log.debug(f"message FAILED {message}")
matrix = Matrix(appserv, spring_lobby_client, config)
appserv.matrix_event_handler(matrix.handle_event)
await matrix.wait_for_connection()
await matrix.init_as_bot()
# appservice_account = await appserv.intent.whoami()
# user = appserv.intent.user(appservice_account)
await appserv.intent.set_presence(PresenceState.ONLINE)
# location = config["homeserver"]["domain"].split(".")[0]
# external_id = "MatrixAppService"
# external_username = config["appservice"]["bot_username"].split("_")[1]
# for room in rooms:
#
# enabled = config["bridge.rooms"][room]["enabled"]
# room_id = config["bridge.rooms"][room]["room_id"]
# room_alias = f"{config['appservice.namespace']}_{room}"
#
# if enabled is True:
# await user.ensure_joined(room_id=room_id)
# await appserv.intent.add_room_alias(room_id=RoomID(room_id), alias_localpart=room_alias, override=True)
# # else:
# # # await appserv.intent.remove_room_alias(alias_localpart=room_alias)
# # try:
# # await user.leave_room(room_id=room_id)
# # except Exception as e:
# # log.debug(f"Failed to leave room, not previously joined: {e}")
appserv.ready = True
log.info("Initialization complete, running startup actions")
for signame in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(
getattr(signal, signame),
lambda: asyncio.ensure_future(spring_lobby_client.exit(signame)))
if logger != "" : self.logger = logger
#runtime
if debug_level : self.debug_level = debug_level
if sock_timeout : self.sock_timeout = sock_timeout
for group in groups_list:
self.groups.append(Group(group,self.parser))
for work in works_list:
self.works.append(Works(work,self.parser))
return self
class TaskServerLog:
@staticmethod
def get_logger(task_ser_config):
vavava.util.assure_path("./log")
logging.config.fileConfig(task_ser_config.log_config_file)
return logging.getLogger(task_ser_config.logger)
if __name__ == "__main__":
config = TaskServerConfig()
config.load(("task_server.conf"))
log = TaskServerLog.get_logger(config)
log.debug("testing ...............")
"--config",
type=str,
default="config.yaml",
metavar="<path>",
help="the path to your config file")
parser.add_argument("-b",
"--base-config",
type=str,
default="example-config.yaml",
metavar="<path>",
help="the path to the example config "
"(for automatic config updates)")
args = parser.parse_args()
config = Config(args.config, args.base_config)
config.load()
config.update()
logging.config.dictConfig(copy.deepcopy(config["logging"]))
init_log_listener()
log = logging.getLogger("maubot.init")
log.info(f"Initializing maubot {__version__}")
loop = asyncio.get_event_loop()
init_zip_loader(config)
db_session = init_db(config)
clients = init_client_class(db_session, loop)
plugins = init_plugin_instance_class(db_session, config, loop)
management_api = init_mgmt_api(config, loop)
server = MaubotServer(config, loop)
def main(heartbeat_stop_callback=None):
global config
global home_dir
parser = OptionParser()
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
help="verbose log output",
default=False)
parser.add_option(
"-e",
"--expected-hostname",
dest="expected_hostname",
action="store",
help=
"expected hostname of current host. If hostname differs, agent will fail",
default=None)
parser.add_option("--home",
dest="home_dir",
action="store",
help="Home directory",
default="")
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
home_dir = options.home_dir
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(),
logging_level)
global is_logger_setup
is_logger_setup = True
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(),
logging_level)
Logger.initialize_logger('resource_management',
logging_level=logging_level)
if home_dir != "":
# When running multiple Ambari Agents on this host for simulation, each one will use a unique home directory.
Logger.info("Agent is using Home Dir: %s" % str(home_dir))
# use the host's locale for numeric formatting
try:
locale.setlocale(locale.LC_ALL, '')
except locale.Error as ex:
logger.warning(
"Cannot set locale for ambari-agent. Please check your systemwide locale settings. Failed due to: {0}."
.format(str(ex)))
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(
config.get('agent', 'data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
update_open_files_ulimit(config)
if not config.use_system_proxy_setting():
logger.info('Agent is configured to ignore system proxy settings')
reconfigure_urllib2_opener(ignore_system_proxy=True)
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
#
# Iterate through the list of server hostnames and connect to the first active server
#
active_server = None
server_hostnames = hostname.server_hostnames(config)
connected = False
stopped = False
# Keep trying to connect to a server or bail out if ambari-agent was stopped
while not connected and not stopped:
for server_hostname in server_hostnames:
server_url = config.get_api_url(server_hostname)
try:
server_ip = socket.gethostbyname(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)',
server_url, server_ip)
except socket.error:
logger.warn(
"Unable to determine the IP address of the Ambari server '%s'",
server_hostname)
# Wait until MAX_RETRIES to see if server is reachable
netutil = NetUtil(config, heartbeat_stop_callback)
(retries, connected,
stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger)
# if connected, launch controller
if connected:
logger.info('Connected to Ambari server %s', server_hostname)
# Set the active server
active_server = server_hostname
# Launch Controller communication
run_threads(server_hostname, heartbeat_stop_callback)
#
# If Ambari Agent connected to the server or
# Ambari Agent was stopped using stop event
# Clean up if not Windows OS
#
if connected or stopped:
ExitHelper().exit(0)
logger.info("finished")
break
pass # for server_hostname in server_hostnames
pass # while not (connected or stopped)
return active_server
'''
Created on Nov 12, 2018
@author: colin
'''
#===============================================================================
# Setup Logging
#===============================================================================
import logging
import logging.config
logger = logging.getLogger('Main')
LEVEL = 'INFO'
logging.basicConfig(format='%(asctime)s %(name)s:%(levelname)s:%(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
level=LEVEL)
from CardStack import CardViewer
from CardStack import CardConfig
config = CardConfig('config')
message = config.info()
load = config.load(
'/Users/colin/Documents/eclipse-workspace/JobCard3/config/config.yaml')
write = config.write('/tmp/output.yaml')
print(config.keys())
build = config.build()