def prepare():
import zmq
from cc_server.commons.configuration import Config
from cc_server.commons.helper import close_sockets
from cc_server.services.web.request_handler import RequestHandler
config = Config()
context = zmq.Context()
logger_socket = context.socket(zmq.PUSH)
logger_socket.connect(config.server_log['external_url'])
tee = logger_socket.send_string
master_socket = context.socket(zmq.PUSH)
master_socket.connect(config.server_master['external_url'])
atexit.register(close_sockets, [logger_socket, master_socket])
global request_handler
request_handler = RequestHandler(
config=config,
tee=tee,
master=master_socket
)
tee('Started service web with pid {}'.format(os.getpid()))
return config
def main():
config = Config()
# start zmq server
context = zmq.Context()
logger_socket = context.socket(zmq.PULL)
logger_socket.bind('tcp://{}:{}'.format(config.server_log['bind_host'],
config.server_log['bind_port']))
# create folder if not existent
log_path = None
if config.server_log.get('log_dir'):
log_dir = os.path.expanduser(config.server_log['log_dir'])
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, FILE_NAME)
# define possible functions for tee
def stdout_func(message):
print(message)
def file_func(message):
with open(log_path, 'a') as f:
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S |"),
message,
file=f)
def both_func(message):
print(message)
file_func(message)
def neither_func(_):
pass
# choose function for tee
suppress_stdout = config.server_log.get('suppress_stdout')
tee = stdout_func
if suppress_stdout:
tee = neither_func
if log_path:
tee = file_func
elif log_path:
tee = both_func
# inform at exit
def at_exit():
tee('Stopped service logs with pid {}'.format(os.getpid()))
atexit.register(at_exit)
# log status
tee('Started service logs with pid {}'.format(os.getpid()))
tee('Loaded TOML config from {}'.format(config.config_file_path))
# start endless loop
while True:
m = logger_socket.recv_string()
tee(m)
def main():
config = Config()
options = {
'bind': '{}:{}'.format(
config.server_files['bind_host'],
config.server_files['bind_port']
),
'workers': config.server_files.get('num_workers', cpu_count()),
'worker_class': 'gevent'
}
WebApp(app_module='cc_server.services.files.wsgi', options=options).run()
def main():
from cc_server.commons.configuration import Config
from cc_server.commons.gunicorn_integration import WebApp
config = Config()
options = {
'bind': '{}:{}'.format(
config.server_web['bind_host'],
config.server_web['bind_port']
),
'workers': config.server_web.get('num_workers', cpu_count()),
'worker_class': 'gevent'
}
WebApp(app_module='cc_server.services.web.wsgi', options=options).run()
def main():
config = Config()
log = Process(target=log_main)
log.daemon = True
log.start()
master = Process(target=master_main)
master.daemon = True
master.start()
if config.server_files:
files = Process(target=files_main)
files.daemon = True
files.start()
web_main()
def prepare():
config = Config()
global tee
global input_files_dir
global result_files_dir
context = zmq.Context()
logger_socket = context.socket(zmq.PUSH)
logger_socket.connect(config.server_log['external_url'])
tee = logger_socket.send_string
atexit.register(close_sockets, [logger_socket])
input_files_dir = os.path.expanduser(config.server_files['input_files_dir'])
result_files_dir = os.path.expanduser(config.server_files['result_files_dir'])
tee('Started service files with pid {}'.format(os.getpid()))
return config
def main():
config = Config()
# start zmq server
context = zmq.Context()
master_socket = context.socket(zmq.PULL)
master_socket.bind('tcp://{}:{}'.format(config.server_master['bind_host'],
config.server_master['bind_port']))
# connect to logger
logger_socket = context.socket(zmq.PUSH)
logger_socket.connect(config.server_log['external_url'])
tee = logger_socket.send_string
# initialize singletons
mongo = Mongo(config=config)
state_handler = StateHandler(config=config, tee=tee, mongo=mongo)
cluster_provider = DockerProvider(config=config, tee=tee, mongo=mongo)
cluster = Cluster(config=config,
tee=tee,
mongo=mongo,
state_handler=state_handler,
cluster_provider=cluster_provider)
scheduler = Scheduler(config=config,
tee=tee,
mongo=mongo,
state_handler=state_handler,
cluster=cluster)
worker = Worker(config=config,
tee=tee,
mongo=mongo,
state_handler=state_handler,
cluster=cluster,
scheduler=scheduler)
# inform at exit
def at_exit():
tee('Stopped service master with pid {}'.format(os.getpid()))
atexit.register(at_exit)
# log status
tee('Started service master with pid {}'.format(os.getpid()))
# start endless loop
while True:
d = master_socket.recv_json()
action = d.get('action')
if action == 'schedule':
pass
worker.schedule()
elif action == 'container_callback':
pass
worker.container_callback()
elif action == 'data_container_callback':
pass
worker.data_container_callback()
elif action == 'update_node_status':
node_name = d.get('data', {}).get('node_name')
if node_name:
pass