def main(test_conf=None):
"""
The entrypoint for the zoe-master script.
:return: int
"""
config.load_configuration(test_conf)
args = config.get_conf()
log_args = {
'level': logging.DEBUG if args.debug else logging.INFO,
'format': LOG_FORMAT
}
if args.log_file != "stderr":
log_args['filename'] = args.log_file
logging.basicConfig(**log_args)
ret = _check_configuration_sanity()
if ret != 0:
return ret
log.info("Initializing DB manager")
state = SQLManager(args)
try:
zoe_master.backends.interface.initialize_backend(state)
except ZoeException as e:
log.error('Cannot initialize backend: {}'.format(e.message))
return 1
metrics = StatsManager(state)
metrics.start()
log.info("Initializing scheduler")
scheduler = getattr(zoe_master.scheduler, args.scheduler_class)(state, args.scheduler_policy, metrics)
restart_resubmit_scheduler(state, scheduler)
log.info("Starting ZMQ API server...")
api_server = APIManager(metrics, scheduler, state)
if config.get_conf().gelf_listener != 0:
gelf_listener = GELFListener()
else:
gelf_listener = None
try:
api_server.loop()
except KeyboardInterrupt:
pass
except Exception:
log.exception('Fatal error in API loop')
finally:
scheduler.quit()
api_server.quit()
zoe_master.backends.interface.shutdown_backend()
metrics.quit()
if gelf_listener is not None:
gelf_listener.quit()
def zoe_web_main(test_conf=None) -> int:
"""
This is the entry point for the Zoe Web script.
:return: int
"""
config.load_configuration(test_conf)
args = config.get_conf()
log_args = {
'level': logging.DEBUG if args.debug else logging.INFO,
'format': LOG_FORMAT
}
if args.log_file != "stderr":
log_args['filename'] = args.log_file
logging.basicConfig(**log_args)
logging.getLogger("MARKDOWN").setLevel(logging.WARNING)
logging.getLogger("tornado").setLevel(logging.WARNING)
if config.get_conf(
).auth_type == 'ldap' and not zoe_api.auth.ldap.LDAP_AVAILABLE:
log.error(
"LDAP authentication requested, but 'pyldap' module not installed."
)
return 1
sql_manager = zoe_lib.state.SQLManager(config.get_conf())
sql_manager.init_db()
master_api = zoe_api.master_api.APIManager()
api_endpoint = zoe_api.api_endpoint.APIEndpoint(master_api, sql_manager)
app_settings = {
'static_path':
os.path.join(os.path.dirname(__file__), "web", "static"),
'template_path':
os.path.join(os.path.dirname(__file__), "web", "templates"),
'cookie_secret':
config.get_conf().cookie_secret,
'debug':
args.debug
}
app = Application(
zoe_api.web.web_init(api_endpoint) +
zoe_api.rest_api.api_init(api_endpoint), **app_settings)
JinjaApp.init_app(app)
log.info("Starting HTTP server...")
http_server = HTTPServer(app)
http_server.bind(args.listen_port, args.listen_address)
http_server.start(num_processes=1)
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("CTRL-C detected, terminating")
return 0
def main():
"""The main entrypoint function."""
conf = load_configuration()
config.load_configuration(conf)
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logging.getLogger('kazoo').setLevel(logging.WARNING)
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('docker').setLevel(logging.INFO)
logging.getLogger("tornado").setLevel(logging.DEBUG)
state = FakeSQLManager()
zapp_description = json.load(args.jsonfile)
print('Validating zapp description...')
zoe_lib.applications.app_validate(zapp_description)
exec_id = state.execution_new('test', 'fake_user', zapp_description)
e = state.execution_list(only_one=True, id=exec_id)
_digest_application_description(state, e)
print('Zapp digested, starting containers...')
execution_to_containers(e)
print('Giving the containers a few seconds to start...')
time.sleep(5)
swarm = SwarmClient(args)
for service in e.services:
print("Service {}, docker ID: {}".format(service.name, service.docker_id))
logs = swarm.logs(service.docker_id, False)
logs = logs.decode('utf-8').split('\n')
for log_line in logs[-10:]:
print(log_line)
print("Execution as been started, press CTRL-C to terminate it")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
print('Terminating...')
terminate_execution(e)
def zoe_web_main() -> int:
"""
This is the entry point for the Zoe Web script.
:return: int
"""
config.load_configuration()
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
if config.get_conf().auth_type == 'ldap' and not zoe_api.auth.ldap.LDAP_AVAILABLE:
log.error("LDAP authentication requested, but 'pyldap' module not installed.")
return 1
zoe_api.db_init.init()
api_endpoint = zoe_api.api_endpoint.APIEndpoint()
app_settings = {
'static_path': os.path.join(os.path.dirname(__file__), "web", "static"),
'template_path': os.path.join(os.path.dirname(__file__), "web", "templates"),
# 'debug': args.debug
}
app = Application(zoe_api.web.web_init(api_endpoint) + zoe_api.rest_api.api_init(api_endpoint), **app_settings)
JinjaApp.init_app(app)
log.info("Starting HTTP server...")
http_server = HTTPServer(app)
http_server.bind(args.listen_port, args.listen_address)
http_server.start(num_processes=1)
retry_cb = PeriodicCallback(api_endpoint.retry_submit_error_executions, 30000)
retry_cb.start()
retry_cb = PeriodicCallback(api_endpoint.cleanup_dead_executions, 60000)
retry_cb.start()
try:
IOLoop.current().start()
except KeyboardInterrupt:
print("CTRL-C detected, terminating")
return 0
def main():
"""
The entrypoint for the zoe-master script.
:return: int
"""
config.load_configuration()
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
if config.get_conf().influxdb_enable:
metrics = InfluxDBMetricSender(config.get_conf().deployment_name, config.get_conf().influxdb_url, config.get_conf().influxdb_dbname)
else:
metrics = LogMetricSender(config.get_conf().deployment_name)
log.info("Initializing DB manager")
state = SQLManager(args)
log.info("Initializing scheduler")
scheduler = ZoeScheduler()
monitor = ZoeMonitor(state)
checker = ZoeSwarmChecker(state)
restart_resubmit_scheduler(state, scheduler)
log.info("Starting ZMQ API server...")
api_server = APIManager(metrics, scheduler, state)
try:
api_server.loop()
except KeyboardInterrupt:
pass
except Exception:
log.exception('fatal error')
finally:
scheduler.quit()
monitor.quit()
checker.quit()
api_server.quit()
metrics.quit()
def main():
"""
The entrypoint for the zoe-master script.
:return: int
"""
config.load_configuration()
args = config.get_conf()
if args.debug:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
if config.get_conf().influxdb_enable:
metrics = InfluxDBMetricSender(config.get_conf().deployment_name, config.get_conf().influxdb_url, config.get_conf().influxdb_dbname)
else:
metrics = LogMetricSender(config.get_conf().deployment_name)
log.info("Initializing DB manager")
state = SQLManager(args)
log.info("Initializing scheduler")
scheduler = ZoeScheduler()
monitor = ZoeMonitor(state)
restart_resubmit_scheduler(state, scheduler)
log.info("Starting ZMQ API server...")
api_server = APIManager(metrics, scheduler, state)
try:
api_server.loop()
except KeyboardInterrupt:
pass
except Exception:
log.exception('fatal error')
finally:
scheduler.quit()
monitor.quit()
api_server.quit()
metrics.quit()
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Create the DB tables needed by Zoe. This script is used in the CI pipeline to prevent race conditions with zoe-api automatically creating the tables while zoe-master is starting at the same time."""
import sys
import time
import zoe_lib.config as config
import zoe_lib.state.sql_manager
config.load_configuration()
print("Warning, this script will delete the database tables for the deployment '{}' before creating new ones".format(config.get_conf().deployment_name))
print("If you are installing Zoe for the first time, you have nothing to worry about")
print("Sleeping 5 seconds before continuing, hit CTRL-C to stop and think.")
try:
time.sleep(5)
except KeyboardInterrupt:
print("Aborted.")
sys.exit(1)
zoe_lib.state.sql_manager.SQLManager(config.get_conf()).init_db(force=True)
def mock_config(self, zoe_configuration): # pylint: disable=redefined-outer-name
"""Fixture for mock config method."""
zoe_configuration.max_core_limit = 32
zoe_configuration.max_memory_limit = 64
load_configuration(zoe_configuration)
